o i@sddlmZddlZddlmZddlmZddlm Z m Z ddl m Z m Z mZmZddlmZeeZGdd d ZGd d d eZdS) ) getLoggerN)Fusion)AttentionMaskFormat) FusionUtils NumpyHelper) NodeProto TensorProtohelper numpy_helper) OnnxModelc@sReZdZdZdefddZdefddZdd Zd d Z d e d e dBfddZ dS) AttentionMask: Fuse Attention subgraph into one Attention node. modelcCs2||_i|_i|_t||_tj|_||_ dSN) r mask_indice mask_castedrutilsrZ MaskIndexEnd mask_formatZget_opset_version opset_version)selfrrc/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_attention.py__init__s  zAttentionMask.__init__rcCs ||_dSr)r)rrrrrset_mask_format s zAttentionMask.set_mask_formatcCs*||jvr||j|ksJ||j|<dSr)r)rmask mask_indexrrrset_mask_indice#s zAttentionMask.set_mask_indicecCs t|jdks Jtt|jSNr)lenrnextiter)rrrrget_first_mask(szAttentionMask.get_first_maskmask_2dreturnNc Cs`|jtjkrdS||jvr|j|S|j|r!|j|\}}n |j|\}}d}|r2||j |<|jtj kr?||j|<|S|j d}|j dkrlt jd|g|g|j ddd}|jt ddgt d d gn7d }|j|dur|jt j|tjdgdgd d t jd||g|g|j ddd}|jt d d g|j|||j|<|S)NTr Z ReduceSumZ MaskReduceSuminputsoutputsnameaxesZkeepdimsrZort_const_1_reduce_sum_axesFr( data_typedimsvalsraw)rrZNoMaskrrZfind_graph_inputrZcast_graph_input_to_int32Zcast_input_to_int32rr create_node_namerr make_node attributeextendmake_attributeget_initializeradd_initializerZ make_tensorrINT64add_node)rr"ZcastedZ input_nameZ _cast_nodeZ output_nameZmask_index_nodeZ axes_namerrr process_mask,sV          $    zAttentionMask.process_mask) __name__ __module__ __qualname____doc__r rrrrr!strr9rrrrr s r c(s,eZdZdZdddddgfdededed edBd ed ed ee ffd d Z de de eeffddZ de de eeffddZde fddZde fddZde de de fddZde d e d!e fd"d#Zd$e d%e dBd&e dBd'e de dBf d(d)Zd*e d+e d,e d$e d%e dBd&e dBde e e e ffd-d.Z / /  / / / / dAd*e d+e e BdBd,e e BdBd$e d%e dBd&e dBdeded0e d1e de d2ede de d3e d4e d5ede dBf$d6d7Z /  / / / / dBd8e dBd*e d+e d,e d$e d%e d&e deded9e d0e d:e d;ede de d3e d4e dZd?d@ZZS)CFusionAttentionr NFSkipLayerNormalizationLayerNormalizationr hidden_size num_headsattention_maskuse_multi_head_attention!disable_multi_head_attention_biassearch_op_typesc sh|rdnd}t|||||_||_|r|nt||_||_||_d|_d|_ d|_ d|_ d|_ dS)NMultiHeadAttention AttentionT) superrrBrCr rDrErFmask_filter_valuenum_heads_warninghidden_size_warning shape_infershape_infer_done) rrrBrCrDrErFrGZattention_op_name __class__rrrls  zFusionAttention.__init__concatr#cCst|jdkr;|j|jd}|j|jd}t|tjr;|jdkr;t|tjr;|jdkr;|d|d|dfS|j|j fS)aU Detect num_heads and hidden_size from Concat node in the following subgraph: SkipLayerNormalization or EmbedLayerNormalization / | MatMul Shape | | Add Gather(indices=0) | | | Unsqueeze | | | Concat (*, -1, 12, 64) | / Reshape | Transpose r*r) rinputrget_constant_value isinstancenpndarraysizerCrB)rrRrC head_sizerrr)get_num_heads_and_hidden_size_from_concats     z9FusionAttention.get_num_heads_and_hidden_size_from_concat reshape_qcCs*|j|jd}|dur1|j|d}|dur"|jdkr"||Std|jd|j|j fSt |t j rIt |dksI|ddksI|ddkrUtd ||j|j fS|d}|d}||}|jdkry||jkry|jrytd |j|d |_|j dkr||j kr|jrtd |j |d |_||fS) zDetect num_heads and hidden_size from a reshape node. Args: reshape_q (NodeProto): reshape node for Q Returns: Tuple[int, int]: num_heads and hidden_size r*NConcatz%s is not initializer.rSrTrrUzGq_shape_value=%s. Expected value are like [0, 0, num_heads, head_size].z>--num_heads is %d. Detected value is %d. Using detected value.Fz@--hidden_size is %d. Detected value is %d. Using detected value.)rrWrVZ get_parentop_typer]loggerdebugrCrBrXrYrZrrLwarningrM)rr^Z q_shape_valuerRrCr\rBrrrget_num_heads_and_hidden_sizes:       z-FusionAttention.get_num_heads_and_hidden_sizeadd_qkcCs|js|jjdd|_d|_|jdurdS|j|jd}|j|jd}|dus/|dur7td|dS||krCtd|dS|jdS)NT)updaterr*zone of the inputs of %s is Nonez)the shape of two inputs of %s is not same)rOrZinfer_runtime_shaperNZget_edge_shaperVrarb)rreZ input_0_shapeZ input_1_shaperrrget_add_qk_strs    zFusionAttention.get_add_qk_strcsdttfdd|j}t|dkrSt|dks J|jd}tjdfddt|j Dg|dd }|j ||j |j |<S) NZ_maskcs|jdkSr)output)node)mask_output_namerrsz0FusionAttention.reshape_add_qk..r*rr_csg|]}qSrr).0_)rerr sz2FusionAttention.reshape_add_qk..r&r'r(axis) listfilter nodes_to_addrrr0r r1rangerCappendthis_graph_namenode_name_to_graph_name)rreZ concat_nodeconcat_node_nameZconcat_add_qk_fp32r)rerjrreshape_add_qks     zFusionAttention.reshape_add_qkpast_kpast_vc Cs|jd}|jd}|ddd}|ddd}tjd|g|g|dgd}tjd|g|g|dgd}|j||j||j|j|<|j|j|<|jd} |dd ddd d } tjd||g| g| dd } |j| |j|j| <| S) zConcatenate past_k and past_v inputs to create past_kv input. Args: past_k (str): name of past K value past_v (str): name of past V value Returns: kv_output_name (str): name of past KV value UnsqueezeZ_5d.rmr)r&r'r(r)r_z.valuez.kv_valueZ_kvro) rr0replacer r1rsrurvrw) rrzr{Zunsqueeze_k_nameZunsqueeze_v_nameZ k_5d_nameZ v_5d_nameZk_5dZv_5drxZkv_output_name concat_kvrrrrsD      zFusionAttention.concat_kvpresent_k_namepresent_v_namekv_nodec Csd\}}|j|}|j|}|dur(tjtjddd|d}|j||j|dur@tjtjddd|d}|j||j|jd}|jd} t j d||g|g|dd } t j d||g|g| dd } |j | |j | |j|j |<|j|j | <dS) a?Split kv_node containing present KV values into separate present K and present V values. Args: present_k_name (str): name of output to store present K value in present_v_name (str): name of output to store present V value in kv_node (str): name of present KV values )Zindex_0Zindex_1NrZint64)Zdtyper(r*ZGatherro)rr5r Z from_arrayrYarrayr6rvr0r r1rsrurw) rrrrZk_indexZv_indexZk_dimZv_dimZ gather_k_nameZ gather_v_name present_k present_vrrrsplit_kv8s:      zFusionAttention.split_kvq_addk_addv_add name_prefixcCs|j|jdp|j|jd}t|}t|}t|}|dur<|j|jdp6|j|jd} t| }|durW|j|jdpQ|j|jd} t| }tj|||fdd} dt|j } |d} |j | |j | g| d| S)Nr*rrprU _qkv_biasr(r,r-r.) rr5rVrto_arrayrYZ zeros_likestackprodshaper6r,)rrrrrq_biasqbkbvbk_biasv_biasqkv_bias qkv_bias_dim bias_namerrrcreate_combined_qkv_biases($   $ $ z(FusionAttention.create_combined_qkv_biasq_matmulk_matmulv_matmulc#Cs"|jd}|jd|jdkr|jd|jdksJ|j|jd}|j|jd} |j|jd} t|} t| } t| } | j| jkrR| j| jksTJ| jd}tj| | | fdd |d|f}|d}|j ||j |jd|jdg|d|d}t j d|jd|g|g|d }|j|j|<|g}|d }|j |tjdgdgd d |d }|j |tjdg|gd d |d}|j |tjdgd|gd d |d}|j |tjdgd|gd d |d}|j |tjdgdgd d |d}t j d||||g|g|jdd }|j|j|j<|d}t j d||||g|g|jdd }|j|j|j<|d}t j d||||g|g|jdd }|j|j|j<|}|} |}!||||g|jr|dur|j|jdrodnd}"tt|j|j|"r||jd|"<|}|||j|j|j<|dur|j|jdrdnd}"tt|j|j|"r||jd|"<|} |||j|j|j<|dur|j|jdrdnd}"tt|j|j|"r||jd|"<|}!|||j|j|j<|j||| |!fS)aCreate packed QKV MatMul node before MultiHeadAttention node. This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path Returns: q_output (NodeProto): Slice node for Q k_output (NodeProto): Slice node for K v_output (NodeProto): Slice node for V MatMulrr*rrU _qkv_weightrZ_qkv_outr%Z_q_start_indexFr+Z_k_start_indexZ_v_start_indexrTZ_end_of_qkv_indexZ_qkv_last_axisZ_q_outSliceZ_k_outZ_v_outN)rr0rVr5rrrrYrZreshaper6r,r r1rvrwrr7r(r3rFanyrurs)#rrrrrrrZmatmul_node_nameq_weightk_weightv_weightqwkwvwd qkv_weightZqkv_weight_nameZqkv_matmul_outputZ qkv_matmul qkv_nodesZ q_slice_nameZ k_slice_nameZ v_slice_nameZend_of_qkv_nameZqkv_last_axis_nameZq_slice_outputq_sliceZk_slice_outputk_sliceZv_slice_outputv_sliceZq_outputZk_outputZv_outputZinitializer_inputrrrcreate_packed_qkv_matmul_nodes ,    "                   z-FusionAttention.create_packed_qkv_matmul_noderhkey_padding_maskunidirectionalrr packed_qkvcCs,|dksJ|dkr||dkrtd||dSdd|jjD}|jd}g}|rM|||||||\}}}||jd|jd|jdgn`t |t r~t |t r~|j rl||jd|jd|jdgnA||jd|jd|jdgn/t |t rt |t r||vr||vr|j r||jd||gn||jd||gndS|j s| ||||}||n|d| r|r|| | | |gn | s| r|| | g| g}|r|r|||gtjd|||d }d |_|jtd || r|jtd t| |d|S) aCreate a MultiHeadAttention node. Args: q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size) k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size) q_add (NodeProto): name of Add from Q path k_add (NodeProto): name of Add from K path v_add (NodeProto): name of Add from V path num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. output (str): output name of MHA key_padding_mask (str): name of key padding mask add_qk (str): name of add after Q x K' unidirectional (bool): whether to apply causal attention mask automatically or not past_k (str): name of past K value - (batch_size, num_heads, past_sequence_length, head_size) past_v (str): name of past V value - (batch_size, num_heads, past_sequence_length, head_size) present_k (str): name of present K value - (batch_size, num_heads, sequence_length, head_size) present_v (str): name of present V value - (batch_size, num_heads, sequence_length, head_size) packed_qkv (bool): whether to combine MatMuls from Q, K, V paths Note: This is for the scenario where an Attention node should be created but cannot be created because past_key and past_value are separate inputs and not one concatenated input. Returns: Union[NodeProto, None]: the node created or None if failed. r9input hidden size %d is not a multiple of num of heads %dNcSsh|]}|jqSrr)rlrirrr HzBFusionAttention.create_multihead_attention_node..rIrrHr% com.microsoftrCr)rarbrgraphrVr0rr3rhrXrrFr>rrur r1domainr2r4intincrease_counter)rrrrrrrrCrBrhrrerrzr{rrrZgraph_input_namesZ mha_node_nameZ mha_inputsrrrrZ mha_outputsZmha_noderrrcreate_multihead_attention_nodesl /  $$$   z/FusionAttention.create_multihead_attention_noder first_input add_qk_strcausalscalec5Csj|dksJ| dkr| |dkrtd| |dSd}|dur)|dur)|dur)d}|j|jd}|j|jd}|j|jd}d\}}}|r|j|jdp\|j|jd}|j|jdpn|j|jd}|j|jdp|j|jd}|r|r|r|sdS|durt|jdddSt|}t|}t|}|j|jksJ|jd}|jd}|jd}||kr|ksJJ| dkr| |krt d | |d} |j|jkrd} t |jdd}!t |jdd}"t |jdd}#d}$| rt j |||fdd }%|!|"|#}$nt j |||fdd }%d |!}$d}&d}'|rt|}(t|})t|}*t |(j}+t |)j},t |*j}-|+|,kr_|!ksbJJ|-|#ksiJ| r}t j |(|)|*fdd }'|+|,|-}&nt j |(|)|*fdd }'d |+}&|jd }.|js|j|.d |j|t|$g|%d|r|j|.d|jt|&g|'d|jr| rtddS|jd|jd|jd|.dg}/|dur|/|tjd|/| g|.d}0|dnr| |.d |r|.dndg}/|dur |/|n|/d|o|}1|1r!|||}2|/|2| r1|1s,|/d|/| | g}3|rT|rT|dddddd}4|3|4||||4tjd |/|3|.d}0|d d|0_|0jtd|g| r~|0jtddg|dur|0jtd|g| r|0jtd|!|"|#gg|jdur|0jtdt |jg|0S)a>Create an Attention node. Args: mask_index (str | None): mask input q_matmul (NodeProto): MatMul node in fully connection for Q k_matmul (NodeProto): MatMul node in fully connection for K v_matmul (NodeProto): MatMul node in fully connection for V q_add (NodeProto): Add bias node in fully connection for Q k_add (NodeProto): Add bias node in fully connection for K v_add (NodeProto): Add bias node in fully connection for V num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning. hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning. first_input (str): first input name output (str): output name add_qk_str (str): name of Add node after Q x K' causal: whether it is uni-directional mask. past_k (str): name of input for past K value past_v (str): name of input for past V value present_k (str): name of output to store present K value present_v (str): name of output to store present V value scale: scale before softmax Returns: Union[NodeProto, None]: the node created or None if failed. rrNTFr*)NNNzl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionzInput hidden size (%d) is not same as weight matrix dimension of q,k,v (%d). Please provide a correct input hidden size or pass in 0rrUrIrrrzVMultiHeadAttention does not support relative_position_bias: cannot fuse the attention.rHr%rz.key_keyr}rmrrCrrZqkv_hidden_sizesrK)!rarbrr5rVprintrrrrcrYrZ concatenaterr0rEr6r,rrhrur r1rrrrrr2r3r4rKfloat)5rrrrrrrrrCrBrrhrrrzr{rrrZhas_biasrrrrrrrrrZ qw_in_sizeZ kw_in_sizeZ vw_in_sizeZis_qkv_diff_dimsZ qw_out_sizeZ kw_out_sizeZ vw_out_sizeZqkv_weight_dimrrrrrrZ q_bias_shapeZ k_bias_shapeZ v_bias_shapeZattention_node_nameZattention_inputsZattention_nodeZ past_existsZpast_kvZattention_outputsZ present_kvrrrcreate_attention_nodes . $$$                                 z%FusionAttention.create_attention_nodec;CsV|}|}|jdkr|j|dd}|dur|}ndS|j|gdgd}d}|dur4|\} } } } } n|j|gdgd}|durK|\} }} } ndSg} t|jD]\}}||vr]qT||djdkrgqT| |qTt| dkrudS| d}|j|d d}|dur||jd}|durt|d kr|d}|jdkr|jd}n,dS|durt|d kr|jd}ndS|jdkr||}|D] }|jdkr|jd}q||}|jd krt|jd kr|jd}||}dd|D}| ddkrdS|j| gdgd}|durt ddS|\} } }}d}d}d}d}gdgdfgdgdfgdgdfgdgdfgdgdfgdgd fd!}d} | D]6\}!}"|j| |"d|"d} | durjqS|!d"krrd#}n|!d$krzd#}n|!d%krd#}n|!d&krd#}| durt d'dSd}#d}$d}%d}&|r| \} }%}$} n$|r| \} }#}%}$n|r| \} } }$n|r| \} }#}$}&} n| \} }#} }$|&p|$}&|j|&gdgd(}'|'dur|j|&gd)gd*}'|'durt d+dS|'d,}(|'d-})|'d.}*|$}+|r'|j|$d d/gddg},|,dur#t d0dS|,\}+} |j|+gd|r3dnddddg}-|-durW|j|$gd1gd2}-|-durWt d3dS|-d-}.|-d.}/d}0d4}1|r|j |%gd5gdfgd6gdfgd7gd8fg|\} }0} n_|r|j |%gd9gd8fgd6gdfg|\} }0} |#dur||#}1|1durt d:|#dSn+|rn'|j |#gd;gd<fgd=gd>fgd?gd@fgdAgdBfg|\} }0} |s|0durt dCdS|s0t|0dkr0|j|0d\} }2|2dus"t|2tjr"|2jdkr"t|2dkr$dSt|2dDkr0t|2|_|jd|kr%|*jd|kr'|/jd|kr)|sV|j|0d.jdnd}3|dur_| n| }4||(\}5}6|5dksr|6dkryt dEdS|j|3|*|/||)|.||5|6||4jd|1dF }7|7durdS|j|7|j|j|7j<|dur|jd}8dG|8}9|jdH|8t j!d gdd|5t"|6|5gddI}:|j#t$%dJ|4jd|:jg|9gdK|8|j|9|jd<|j&'|4| | g|j&'| |j&'|j(s|'n|'dd.|j&'|j(s |-n|-dd.|j&'|j(s|n|dd.d#|_)dSdSdSdS)LNrAAddr)rrReshape Transposer)NNrrr)rZEinsumrr)r*Nrrr*MulrTr@rScSsg|]}|jqSr)r`)rlchildrrrrnrz(FusionAttention.fuse..rrU)rrrr)r*rrNz&fuse_attention: failed to match v pathF)SoftmaxrDivr)rrNr)rrrr)rWhererr)rrrTr)rrrr)rrrrT)rrr)rrr)rrrrSqrt)rrNrr*)Zpath1Zpath2path3path4path5sdparTrrrz'fuse_attention: failed to match qk path)rrrN)rrrrr)rrrrNz&fuse_attention: failed to match q pathrrz/fuse_attention: failed to match mul sqrt q path)rrrrr)r*rrrNz&fuse_attention: failed to match k pathr)ExpandrEqual)rr|r|)Castrrr)rrrr)rrr|r|z6fuse_attention: failed to verify shape inference of %s)rSubrr|r|)Nrr*rr)rrr|r|)Nrr*r)rrrrr|r|)Nrrr*rr)rrrrrr|r|)Nrrr*rrrz)fuse_attention: failed to match mask pathizmFailed to detect num_heads and hidden_size for Attention fusion. Please specify those parameters in argument.) rrrrrrrrCrBrrhrZedge_modified_Zshape_modified_tensorr+rZreshape_modified_)*r`rZ match_parentZmatch_parent_path enumeraterVrhrurcountrarbitemsZmatch_parent_pathsrgZget_constant_inputrXrYrZr[rrKrDr9rdrcrrsrvrwr(r6rr7rr8r r1Znodes_to_remover3rEZ prune_graph);rriZinput_name_to_nodesZoutput_name_to_nodeZnormalize_nodeZ start_nodeZadd_before_layernormrZ einsum_nodermZ reshape_qkvZ transpose_qkvZ matmul_qkvZ other_inputsZ_iZ node_inputZ root_inputZmul_before_layernormZ mul_childrenZlayernorm_nodechildrenrZ parent_nodeZchildren_typesZv_nodesZadd_vZmatmul_vZ is_distillZis_distill_addZis_no_mask_attentionZis_sdpaZqk_pathsZqk_nodeskvreZ matmul_qkZwhere_qkZafter_qZq_nodesr^Zadd_qZmatmul_qZafter_kZ mul_k_nodesZk_nodesZadd_kZmatmul_kZ mask_nodesrZmul_valrZattention_last_nodeZ q_num_headsZ q_hidden_sizenew_nodeZ unique_indexZnew_edgeZ shape_tensorrrrfusevs                                             0            zFusionAttention.fuse)rrFrrrrF)rFrrrrN)r:r;r<r=r rr boolrqr>rrtupler]rdrgryrrrrrrrr __classcell__rrrPrr?gs( .7-                 kr?)loggingrnumpyrYZ fusion_baserZfusion_optionsrZ fusion_utilsrrZonnxrrr r Z onnx_modelr r:rar r?rrrrs    V