o iS@sddlZddlZddlZddlmZddlmZddlZddl Z ddl Z ddl m Z ddl mZddl mZmZddlmZddlmZdd lmZdd lmZmZmZmZdd lmZeeZ Gd d d e j!j"Z#dS)N)chain)Path)convert_float_to_float16)RepeatedCompositeFieldContainer) ModelProtoValueInfoProto) OnnxModel)PastKeyValuesHelper) WhisperConfig)convert_inputs_for_ortget_model_dynamic_axesget_sample_decoder_inputsgroup_past_key_values)InferenceSessioncseZdZdZd5dedejjdede ffdd Z d6d ej d ej d Bd e e ej d Bfd dZ d6d ej d ej d Bd e e ej d BfddZ d6d ej d ej d Bd e e ej d BfddZddZddZddZd5de de de fddZd7dede d e fd!d"Zd5d#ed e fd$d%Zdefd&d'Zdede fd(d)Z *   *  *d8d+ed,ed-e d.e de de d/e d0e fd1d2Zd+ed,ede de fd3d4ZZS)9WhisperDecoderz/A Whisper decoder with optional past key valuesFconfigmodel model_implno_beam_search_opcst||_|j|_||_||_|dkrdn|jj|_|dkr#dn|j|_|dkr-|nd|_|jj |_ |jj |_ |jj |j |_ dSNZopenai)super__init__rdevicerrrdecoderproj_outmax_source_positionsZdecoder_attention_heads num_headsZd_model head_size)selfrrrr __class__q/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/models/whisper/whisper_decoder.pyr%s   zWhisperDecoder.__init__Ndecoder_input_idsencoder_hidden_statespast_key_valuesc CsJ|j|||dd}||j}|j}|dur||fSt|\}}||fS)NT)r$ input_idsr%Z use_cache)rrZlast_hidden_stater%r Zgroup_by_self_and_cross) rr#r$r%Zoutputslogitspresent_key_values present_self present_crossr!r!r" hf_forward4s zWhisperDecoder.hf_forwardcs&i}|dur`t|\}}dd|D}dd|D}dd|D}dd|D}tjjjD]0\}}|d|||jj<|d|d||jj<|d|||jj<|d|d||jj<q/j \} } jj|||d} |durjjjD]0}t j ||jj| |jjgdd  | |jj<t j ||jj| |jjgdd  | |jj<qygg} } jjjD]*}| | |jj| | |jj|dur| | |jj| | |jjqfd d| D} fd d| D} | D]}|q|durt| | t| d}| |fS| | fS) NcSg|]}|ddqS transpose.0Zpast_kvr!r!r" Zz.WhisperDecoder.oai_forward..cS*g|]}|g|jdddRqSNr/reshapeshaper2r!r!r"r4[*cSr,r-r0r2r!r!r"r4\r5cSr6r7r9r2r!r!r"r4]r<r/r.)xZxakv_cache)dimc8g|]}|g|jdddjRddqSNr/r8r.r:r;rr1r3Z present_kvrr!r"r4*cr@rArBrCrDr!r"r4rE)r enumeraterrblocksZattnkeyvalueZ cross_attnZinstall_kv_cache_hookstorchcatdetachappendremover Zgroup_by_layerlen)rr#r$r%Z past_kv_cacheself_attn_kv_cachescross_attn_kv_cachesidxblockr>hooksr'r)r*hookr(r!rDr" oai_forwardPs`        zWhisperDecoder.oai_forwardcCs&|jdkr ||||S||||Sr)rrVr+)rr#r$r%r!r!r"forwards zWhisperDecoder.forwardcCs>|jr ddg}|Sddgttddt|jjD}|S)Nr&r$cs4|]}d|d|d|d|fVqdS)Zpast_key_self_Zpast_value_self_Zpast_key_cross_Zpast_value_cross_Nr!r3ir!r!r" s " z-WhisperDecoder.input_names.. first_passlistr from_iterablerangerZdecoder_layers)r input_namesr!r!r"ras    zWhisperDecoder.input_namescCsZ|jrdgttddt|jjD}|Sdgttddt|jjD}|S)Nr'csrX)present_key_self_present_value_self_Zpresent_key_cross_Zpresent_value_cross_Nr!rYr!r!r"r[s z.WhisperDecoder.output_names..css$|] }d|d|fVqdS)rbrcNr!rYr!r!r"r[s r\)r output_namesr!r!r"rds&    zWhisperDecoder.output_namescCs*t|j||}d|vr|js|dd=|S)Nr&r.)r rr)rrard dynamic_axesr!r!r"res zWhisperDecoder.dynamic_axesuse_fp16_inputsuse_int32_inputs return_dictc Csnt|j|jd|jr dnd|jrdnd||d}|r!|jr|d=|S|jr,|d|dfS|d|d|dfS) Nr/rr.)Z batch_sizepast_sequence_lengthsequence_lengthZuse_fp16Z use_int32r%r#r$)r rrr])rrfrgrhinputsr!r!r"rls*   zWhisperDecoder.inputsiois_cross is_outputcCs|jjjjd}d|jvr||j|_|jjjjd}d|jvr4||r-|j|_n|r1dnd|_|jjjjd}d|jvrI||j |_|S)Nr.Z_dim_r/Ztotal_sequence_lengthrj) typeZ tensor_typer;r?Z dim_paramZClearrZ dim_valuerr)rrmrnrorrkrr!r!r"fix_key_value_cache_dimss    z'WhisperDecoder.fix_key_value_cache_dimsio_listcCsg}g}g}|D]D}d|jvrd|jvr||qd|jvr6|j|d|d}|jr0||q||q|j|d|d}|jrG||q||q|jsV|||7}|S)NZpastZpresentrF)rnroT)namerMrrr)rrsroZ reordered_iorPrQrmZnew_ior!r!r"fix_ios$       zWhisperDecoder.fix_iocCs|j|jjdd}t|jjdkr|jjt|jjdks|jj||j|jjdd}t|jjdkrE|jjt|jjdks7|jj||S)NF)rorT)rugraphinputrOpopextendoutput)rrZreordered_inputsZreordered_outputsr!r!r"fix_inputs_and_outputs0s  z%WhisperDecoder.fix_inputs_and_outputscCs|jdkr |r t|}|Sr)rr)rrrfr!r!r"fix_layernorm_weights>sz$WhisperDecoder.fix_layernorm_weightsTonnx_model_pathproviderverboseuse_external_data_formatuse_encoder_hidden_statesuse_kv_cache_inputsc  Cs*|o| |_| o ||_|js|jsJd|j||d} |} |} || | } t|jjdddt J} t j | d}t|jjddd|rO|n|}tjj|| |d| | | dd|d tj||d}||}|||}tj|||dd Wd n1swY|||||d S) alExport decoder to ONNX Args: onnx_model_path (str): path to save ONNX model provider (str): provider to use for verifying parity on ONNX model verbose (bool, optional): print verbose information. Defaults to True. use_external_data_format (bool, optional): use external data format or not. Defaults to False. use_fp16_inputs (bool, optional): use float16 inputs for the KV caches. Defaults to False. use_int32_inputs (bool, optional): use int32 inputs for the decoder_input_ids. Defaults to True. use_encoder_hidden_states (bool, optional): use encoder_hidden_states as model input for decoder-init/decoder-without-past models. Defaults to False. use_kv_cache_inputs (bool, optional): use KV caches as model inputs for decoder-with-past models. Defaults to True. zVOnly one of `use_encoder_hidden_states` and `use_kv_cache_inputs` can be true at once.)rfrgT)parentsexist_okz decoder.onnx) argsfZ export_paramsrardreZ opset_versionZdo_constant_foldingr)Zload_external_data)Zsave_as_external_dataZall_tensors_to_one_fileN)r]Z later_passrlrardrerparentmkdirtempfileTemporaryDirectoryospathjoinrJonnxZexportZ load_modelr{r|rsave verify_onnx)rr}r~rrrfrgrrrlrardreZ tmp_dir_nameZtemp_onnx_model_pathZout_pathrr!r!r" export_onnxJsJ $      zWhisperDecoder.export_onnxcCsJ|j||dd}g}|jr:|jd i|}||d|dD]}|D] } || q*q&n'|jd i|}||d|dD] } || qSt||gd} | dt || } z-t | D]#\} }t || | | }td|dtd t |qxWdSYdS) awVerify ONNX model outputs and PyTorch model outputs match Args: onnx_model_path (str): path to save ONNX model provider (str): execution provider for ONNX model use_fp16_inputs (bool, optional): use float16 inputs for the KV caches use_int32_inputs (bool, optional): use int32 inputs for the decoder_input_ids T)rfrgrhrr.) providersNz Comparing z...z Max diff: r!)rlr]rWrMrLcpunumpyrrunr rFrdnpabsloggerwarningmax)rr}r~rfrgrlZ pt_outputsoutZpresent_key_value_layerZpresent_key_valueZpresent_self_key_valuesessZ ort_outputsrZZ output_namediffr!r!r"rs0  zWhisperDecoder.verify_onnx)F)NN)FF)TFFTFT)__name__ __module__ __qualname____doc__r rJnnModulestrboolrZTensorr^tupler+rVrWrardrerlrrrrrurr{r|rr __classcell__r!r!rr"r"s$  ]   Qr)$loggingrr itertoolsrpathlibrrrrrJZfloat16rZ#google.protobuf.internal.containersrrrZ onnx_modelrZ past_helperr Z transformersr Zwhisper_inputsr r r rZ onnxruntimer getLoggerrrrrrr!r!r!r"s$