o i@sddlZddlZddlZddlmZddlZddlZddlm Z m Z m Z ddl m Z ddlmZmZmZmZddlmZddlmZddlmZdd lmZdd lmZeeZgd Z Gd d d Z!dS)N)Path)add_cache_indirection_to_mhaadd_output_qk_to_mhafix_past_sequence_length)optimize_model) AutoTokenizer WhisperConfigWhisperForConditionalGenerationWhisperProcessor)WhisperDecoder)WhisperEncoder)WhisperEncoderDecoderInit)WhisperJumpTimes)InferenceSession) z whisper-tinyzwhisper-tiny.enz whisper-basezwhisper-base.enz whisper-smallzwhisper-small.enzwhisper-mediumzwhisper-medium.enz whisper-largezwhisper-large-v2zwhisper-large-v3zwhisper-large-v3-turboc@seZdZe  dAdededededef dd Zeded ed ed ed edededededdfddZe   dBdedededej dej deded ede eej j ffddZedeeBeBded ededed ed!ed"ed#efd$d%Ze   &    dCded'ed(ed)ed*ed+eded,ed ed-eded ed efd.d/Ze 0 dDd1ed2ej j dej d3ed4ef d5d6Zed3ed4efd7d8Zed9ejd:ejd;efdedej d3ed4ef d?d@ZdS)E WhisperHelperF output_dirmodel_name_or_pathsuffix new_folderreturncCs^|}tj|rt|jd}n|dd}||7}|r$tj||n|}tj||dS)aBuild onnx path Args: output_dir (str): output directory model_name_or_path (str): pretrained model name, or path to the model checkpoint suffix (str, optional): suffix like "_encoder" or "_decoder_fp16" will be appended to file name. Defaults to None. new_folder (bool, optional): create a new directory for the model. Defaults to False. Returns: str: path of onnx model /z.onnx)ospathisdirrpartssplitjoin)rrrrZ model_name directoryr p/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/models/whisper/whisper_helper.py get_onnx_path*s zWhisperHelper.get_onnx_pathprovider!separate_encoder_and_decoder_inituse_decoder_masked_mha output_qk encoder_path decoder_path cache_dirNc CsXtj||d} | |tj||d} | |tj||d} | ||r(dSddddddiddd d d d gd d diddddd d | jddigii} tj| dd} tt j |dd }| | Wdn1spwYd|vr{gn|iig}| j | jd|dt j || j| j| jddddddd d!d"d#| j| j| jd$ d|dt j || j| j| jd%d%id&d'd(d)| j| j| jd$ | j| jd*| jd+d,d-d.d/| jd0d0d1d1|d/d/d1d/d2d3}|r|d4d5d6d7d8d9|r|d4d5d:d;dNr)Zfeature_extractionsequenceZ operationZ audio_decoderZ AudioDecoder)nametypeZSTFTZSTFTNormi(,?+0?@0B?@*P?g@Y?-b?`h?(&p?xnt?6y?~?gL#?F?gH?gK?g?g%?gbT?gL?g?g?gD?g?g+?g|?g?gG?gA]?g?g?g?g^?gа?g4ӱ?gܲ?g?gD?g9"?gF?g!r?g?g3ܺ?g?g_?g?g\?g?g@X?g8 ?g@þ?g@*v?ga0?g]?g@?gqo?g@r4?g?g!?g?ga?g@3?g?g?g?gb?gk?g4%?gy?g@?gx?g?g@`?g?gJ?g?gR8?gU?g )?gx?g?g8?g@?g@Z?g ?g?g?g`Ă?g?g?g`}?g}?g`E?g}?g?gC}?g?g~?g?gS??g`V@?g?g?g0?g`^A?gw?gx?g]?g!A?g?g8?g?g>?g}?g`1?g`?g 8?g`v?gc?g?g .?gxk?g@է?g?gx?gZ?g@?g?g ?gC?g |?gC?g@f?g%?g'\?g@?g@?g@K?g83?gg?g`R?gw?g?g2?g#d?g?g?g?gu"?g@OP?gr}?g۩?g?gt?g*?gT?g|?gy?g@??g@$?g;?w_?^?j?g?g@?g@Z?g#?g A?g@R^?g,z?g?g?g6?g`]?g@? ?g*#?g 7?gJ?g]]?n?g Q?ӎ?g[??gr??@?#?g?gC?`?_??t??~?rLrKrJrIrHrGgC?g?rFrErDgr?rCgZ?rBgQ?rAg\]?gJ?g7?g@*#?r@g?g@]?g5?g`?g?g+z?gR^?gA?g@#?gZ?g?g@?r?r>r=g;?g$?r<g?gy?g|?gT?g`*?g`t?g@?gک?gq}?gOP?g@u"?g?g?g?g#d?g@2?g@?gw?gR?g@g?g83?gK?g?gྒ?g@'\?g`%?ge?g C?g|?g@C?g` ?g?g?gZ?g`x?g@?gԧ?g xk?g.?g?g@c?gv?g8?g?g0?g`}?g>?g ?g@8?g?g@!A?g]?g@x?g@w?g]A?g 0?g@?g ?g@V@?r;g`S?g?g~?g?g@C}?g?g`}?gE?g}?g}?g`?g?gĂ?g@?g`?g@ ?gY?g?g8?g ?gw?g)?g@U?gR8?g@?g@J?g?g`?g@?gx?g?gy?g3%?gk?g@a?g?g?g?g@3?ga?g?g ?g@?gq4?gpo?g?g\?g@`0?g@)v?g@¾?g7 ?gX?g@?gZ?g?g_?g?g1ܺ?g?gr?gF?g7"?gB?g?gܲ?g2ӱ?gа?gZ?g?g?g?g>]?gC?g?g|?g(?g?gD?g?g?gF?g^T?g%?g?g K?gD?r:gH#?r9r8r7r6r5r4g@Y?r3r2r1r0)n_fftZ frame_length hop_length_comment)r,r-attrsZlog_mel_spectrogramZLogMelSpectrum) chunk_sizerOrNZn_mel)indentzaudio_processor_config.jsonwcpuzonnxruntime-genai)Zlog_idprovider_options input_idszpast_key_self_%dzpast_value_self_%dzpast_key_cross_%dzpast_value_cross_%d)rYZpast_key_namesZpast_value_namesZcross_past_key_namesZcross_past_value_nameslogitszpresent_key_self_%dzpresent_value_self_%d)rZZpresent_key_namesZpresent_value_names) Zsession_optionsfilenameZ head_size hidden_sizeinputsoutputsnum_attention_headsZnum_hidden_layersZnum_key_value_headsZaudio_featuresencoder_hidden_stateszpresent_key_cross_%dzpresent_value_cross_%d)r`Zcross_present_key_namesZcross_present_value_nameswhisper) bos_token_idZcontext_lengthdecoderencoder eos_token_id pad_token_idr- vocab_sizer/FT?rrM)Zdiversity_penaltyZ do_sampleearly_stoppinglength_penalty max_length min_lengthZno_repeat_ngram_size num_beamsnum_return_sequencesZpast_present_share_bufferrepetition_penalty temperatureZtop_kZtop_p)modelsearchrqrcr]past_sequence_lengthcache_indirection)rsrtr^Zoutput_cross_qk_nameszoutput_cross_qk_%dzgenai_config.json)rfrom_pretrainedZsave_pretrainedrr Z num_mel_binsjsondumpsopenrrrwriterbrkbasenameZd_modelZdecoder_attention_headsZdecoder_layersZencoder_attention_headsZencoder_layersrerfrgupdatedump)rr#r$r%r&r'r(rr)config tokenizer processorZaudio_processor_cfgZaudio_processor_jsonfrXZ genai_configr r r!save_processingFs    !.     3K$zWhisperHelper.save_processingT model_impldevicedtypemerge_encoder_and_decoder_initno_beam_search_opcCs|dkr tj||dd}n!ddl} tj|s#|dddd} n|} | j| ||d d }|j |d |dkr@|j |d t j||d } t | |||} d| i} |rgt | |||}| d|int| ||}| || d|rt| ||}| d|i| S)a|Load model given a pretrained name or path, then build models for ONNX conversion. Args: model_name_or_path (str): pretrained model name or path model_impl (str): library to load model from cache_dir (str): cache directory device (torch.device): device to run the model dtype (torch.dtype): dtype to run the model merge_encoder_and_decoder_init (bool, optional): Whether merge encoder and decoder initialization into one ONNX model. Defaults to True. no_beam_search_op (bool, optional): Whether to use beam search op or not. Defaults to False. output_qk (bool, optional): Whether to output QKs to calculate batched jump times for word-level timestamps. Defaults to False. Returns: Dict[str, torch.nn.Module]: mapping from name to modules for ONNX conversion. Zhfeagerr)Zattn_implementationrNrrT)Z download_rootZ in_memoryrrr*rcrd)rdZ decoder_initZ jump_times)r rurarrexistsr load_modelevaltorr r r{r r)rrr)rrrrr&rqraZ name_or_pathr}rc componentsZencoder_decoder_initrdZbatched_jump_timesr r r!rms0  zWhisperHelper.load_modelrqonnx_model_pathverboseuse_external_data_formatuse_fp16_inputsuse_int32_inputsuse_encoder_hidden_statesuse_kv_cache_inputsc Cst|tr||||||dSt|tr!|||||||dSt|tr4|||||||||dSt|trE|||||||dStdt|)a/Export model component to ONNX Args: model (class): PyTorch class to export onnx_model_path (str): path to save ONNX model provider (str): provider to use for verifying parity on ONNX model verbose (bool): print verbose information. use_external_data_format (bool): use external data format or not. use_fp16_inputs (bool): use float16 inputs for the audio_features, encoder_hidden_states, logits, and KV caches. use_int32_inputs (bool): use int32 inputs for the decoder_input_ids. use_encoder_hidden_states (bool): use encoder_hidden_states as model input for decoder-init/decoder-without-past models. use_kv_cache_inputs (bool): use KV caches as model inputs for decoder-with-past models. z%Unknown instance for model detected: N) isinstancer export_onnxr r r ValueErrorr-) rqrr#rrrrrrr r r!rsL     zWhisperHelper.export_onnxrWoptimized_model_path is_float16r_r\num_decoder_layersuse_gpu is_decoderc  Csddlm} | d}d|_|dk|_t|d||d||dd}| r?| r?| r/t|\}}t||}| r?t|tt dd|dd }|j ||dd d S) zHOptimize ONNX model with an option to convert it to use mixed precision.r) FusionOptionsZbartTZrocmF)Z model_typeZ num_headsr\ opt_leveloptimization_optionsrZonly_onnxruntime)Zskip_node_idxs)Zall_tensors_to_one_fileN) Zfusion_optionsrZuse_multi_head_attentionZ!disable_multi_head_attention_biasrrrrlistrangeZsave_model_to_file)rrrr_r\rrrr#rrr%r&rrmZpast_seq_len_namer r r! optimize_onnxs*     zWhisperHelper.optimize_onnxrMrpt_model batch_size prompt_modec s@zddlm}Wn,ty4}z tjd|ddd}td|dt|WYd}~nd}~wwddlm}|d d d d }g} |d krW|dddgddj} n/|dddgddj|dddgddjg} t | |ks{Jt | d| d f} d\} } } }d\}}| || | | |||ddd }|rddg}fdd|D}g}g}t |D]7}t ||j |d|d<| | ||d<|jd i|}|||j|dddq| |d<|d=ng}|jd i|}j|dddg}t|}|d=|d=||||fS)!Nr) load_datasetz.An error occurred while importing `datasets`: T)exc_infozpip install datasetszCCould not import `datasets`. Attempting to install `datasets` via `z`.z)hf-internal-testing/librispeech_asr_dummycleanZ validation)rrMZaudioarraypt)Zreturn_tensors)rRrrMrM)rhrh) input_featuresrkrlrmrnrjrori use_cachezJohn has doubtszMaria has grave doubtscsg|]}|qSr )Zget_prompt_ids.0prr r! LszBWhisperHelper.pt_transcription_for_verify_onnx..r prompt_idsrZskip_special_tokensrirr )Zdatasetsr ExceptionloggererrorwarningrsystemrlentorchcatrrZ from_numpygeneratedetachrWnumpyappend batch_decoder)rrrrrreZ install_cmdZdsZinput_features_rrkrlrmrnrjror]Zpromptsrpt_transcription pt_outputsi pt_outputr rr! pt_transcription_for_verify_onnxsf       z.WhisperHelper.pt_transcription_for_verify_onnxc CsF|dkr|rd}d}d}d}||||h}|Sd}d}d} ||| h}|S) NrMz{ John has doubts whether Sir Frederick Layton's work is really Greek after all and can discover in it but little of Rocky Izy John has doubts whether Sir Frederick Latins work is really Greek after all and can discover in it but little of Rocky Iz Maria has grave doubts whether Sir Frederick Layton's work is really Greek after all and can discover in it but little of Rockyz Maria has grave doubts whether Sir Frederick Latins work is really Greek after all and can discover in it but little of Rocky IzX Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.zY Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.zZ "Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.r ) rrZ'expected_transcription_no_comma_prompt1Z)expected_transcription_misspelled_prompt1Z'expected_transcription_no_comma_prompt2Z)expected_transcription_misspelled_prompt2expected_transcription_optionsZexpected_transcription_no_commaZ!expected_transcription_with_commaZ+expected_transcription_with_quote_and_commar r r!select_transcription_optionscs, z*WhisperHelper.select_transcription_optionsr ort_outputsrcCs||||}}|j|j}}||krKt|dkr!|d}|j}t|dkr.|d}|j}|d|dkrKt|d|d}|d|}|d|}|j|jksSJ||fS)z8Get PyTorch and ONNX Runtime output token ids at index irMrN)shapermin)rrrr ort_outputZpt_shapeZ ort_shapeZmin_lenr r r! get_outputss    zWhisperHelper.get_outputs ort_sessionc&Cstj||dd|}tj||d}tj||d}tj|||||d\} } } } |jg} |jddd}dd |D}| |}d d | D}d d | D}t j t j t j t jt jt jd }d |v}t||ddD]\}}|dkr~| || |<qi|dkrt j|j||d| |<qi|dkrt j||jf||d| |<qi|dkr|s|r| gn|g}t j|||d| |<qig}t|D] }|| |qtdd|D}g}|D]}g||jg|t|}|||qt j|||d| |<qi|dkrt jdg||d| |<qi|dkr%t jddgg||d| |<qi|d kr.cSg|]}|jqSr )r,rentryr r r!rcSrr )r-rr r r!rr)z tensor(float)ztensor(float16)z tensor(int64)z tensor(int32)z tensor(int8)z tensor(uint8)Zextra_decoding_idsF)strictrZ vocab_maskrZprefix_vocab_maskZdecoder_input_idscss|]}t|VqdS)N)rrr r r! sz,WhisperHelper.verify_onnx..Zlogits_processorrMZcross_qk_layer_headrrprhNTr)keyzPyTorch outputs: zONNX Runtime outputs: )(r rurr rrrZdecoder_start_token_idZget_decoder_prompt_idsZ get_inputsnpZfloat32Zfloat16Zint64Zint32Zint8Zuint8ziprrWrZonesrgrrrtolistmaxrfrrepeatrunrrrZallcloserabsrr)&rr)rrrrrrr}r]rrZdecoder_prompt_idsZstart_idrZforced_decoder_idsZ ort_namesZ ort_dtypesZ ort_to_npZuse_extra_decoding_idsr,rZ raw_input_idsZ ort_promptsrmax_lenZpadded_promptsrZ padded_promptrZort_transcriptionrZparityrrZmax_diffdiffZ max_diff_ir r r! verify_onnxs      $ "     zWhisperHelper.verify_onnx)rF)TFF)FFrWFFFF)rMF)__name__ __module__ __qualname__ staticmethodstrboolr"rrrrdictnnModulerr r r rintrr rrrZndarrayrrrr r r r!r)sF   * =  @     0 E   r)"rvloggingrpathlibrrrrZconvert_generationrrrZ optimizerrZ transformersrrr r Zwhisper_decoderr Zwhisper_encoderr Zwhisper_encoder_decoder_initr Zwhisper_jump_timesrZ onnxruntimer getLoggerrrZPRETRAINED_WHISPER_MODELSrr r r r!s"