o ¢ÄiŒã@spddlmZddlmZddlmZddlmZmZm Z ddl m Z ee ƒZ Gdd„deƒZGdd „d eƒZd S) é)Ú getLogger)ÚFusion)Ú FusionUtils)Ú NodeProtoÚ TensorProtoÚhelper)Ú OnnxModelcseZdZdZd2dedef‡fdd„ Zdedd eeefBfd d „Z d ed e ee efde de fdd„Z dd„Zdd„Zdd„Zdd„Zdd„Zdedeed eBffdd„Z  d3ded ed ed!ed"d eBd#ed Bf d$d%„Zd&d'„Zd(d)„Z d4d*d+„Zd,d-„Zd.d/„Zd0d1„Z‡ZS)5ÚFusionEmbedLayerNoMaskzŒ Fuse embedding layer into one node (EmbedLayerNormalization). It supports the following model types: BERT, DistilBert, ALBert. úno maskÚmodelÚ descriptioncs<tƒ |dddg|¡t|ƒ|_d|_d|_d|_d|_dS)NÚEmbedLayerNormalizationÚLayerNormalizationÚSkipLayerNormalizationF)ÚsuperÚ__init__rÚutilsÚ shape_inferÚshape_infer_doneÚ attentionÚ embed_node)Úselfr r ©Ú __class__©úd/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_embedlayer.pyrsü  zFusionEmbedLayerNoMask.__init__ÚaddÚreturnNcCsP|j |dgdg¡}|durdS|j |dgdg¡}|dur dS|d|dfS)NÚGatherré)r Úmatch_parent_path)rrÚ gather_0_pathÚ gather_1_pathrrrÚmatch_two_gather%sz'FusionEmbedLayerNoMask.match_two_gatherÚ layernormÚinput_name_to_nodesÚis_distil_bertc CsÐ|jj|d|dd|_|jdurdS|jd|vrdS||jd}tdd„|Dƒƒ}|gd ¢kr_|D]+}|jd kr^|j |gd ¢gd ¢¡}|dur^|d jd|jdkr^|d|_dSq3t |ƒdkr¶|djdkr¶|djd|vr¶||djd}t |ƒdkr¶|djdkr¶|djd|vr¶||djd} | D] }|jdkr¬||_dSqŸtdd„| Dƒƒ}|rÓ|gd¢krÑ|gd¢krÑ|gd¢krÑt   d¡dSdS|gd¢kræ|gd ¢kræt   d¡dSdS)a§Check that LayerNormalization has a child of Attention node or subgraph like Attention. Args: layernorm (NodeProto): LayerNormalization node input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes is_distil_bert (bool): whether it is DistilBert or not Returns: bool: whether there is Attention node or subgraph like Attention Ú AttentionF)Ú recursiveNTrcSóg|]}|j‘qSr©Úop_type©Ú.0ÚchildrrrÚ JózCFusionEmbedLayerNoMask.check_attention_subgraph..)ÚMatMulr1r1rr)ÚAddr1ÚMultiHeadAttentionr1)NNrréÿÿÿÿérr1r2cSr)rr*r,rrrr/gr0)r1r1r1ÚShaper)r2r1r1r1r6r6)r2r1r1r1r6z Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Mul -- Cast(to=7) -- Add(B=1) -- Cast(to=7)* --> Gather | ^ V | +------------------------------+ Roberta new pattern from transformers v4.9: (input_ids) --> Equal(B=?) -- Not -- Cast(to=6) -- CumSum(axis=1) -- Add(B=0) -- Mul -- Cast(to=7) -- Add(B=1) --> Gather | ^ V | +-------------------------------------------+ start_node = position_embedding_gather start_index = 1 # match optional Cast node. parent = self.model.get_parent(start_node, start_index, output_name_to_node) if parent is None: return if parent.op_type == "Cast": if OnnxModel.get_node_attribute(parent, "to") != 7: return start_node = parent start_index = 0 i, path, return_indices = self.model.match_parent_paths( start_node, [ (['Add', 'Cast', 'Mul', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0]), (['Add', 'Cast', 'Mul', 'Add', 'CumSum', 'Cast', 'Not', 'Equal'], [start_index, 0, 0, 0, 0, 0, 0, 0])], output_name_to_node) if path is not None: # constant input of Add shall be 1. i, value = self.model.get_constant_input(path[0]) if value != 1: return False _, self.padding_word_id = self.model.get_constant_input(path[-1]) return input_ids == path[-1].input[0] Fr©rrIrJrKrrrÚ match_position_embedding_robertaÂs-z7FusionEmbedLayerNoMask.match_position_embedding_robertac Cs˜|j |ddgddg|¡}|durdS|\}}|j |jd¡}|durTt|jƒdkrT|jddkrT|j |ddg¡rT|j |ddg¡rTt|jƒd ksV|j |d dg¡sVdS|j ¡}|d krjt   |d dg¡sidSn |j |ddg¡sudS|j  |d|¡} | durƒdS| j d kr›|j | dd¡s’dS|j  | d|¡} n| } | dus¦| j d kr¨dS|j | dd¡s²dS|j  | d|¡} | dusÃ| j dkrÅdS|| jdkS)a  Match position embedding path from input_ids to Gather for BERT. BERT Embedding Layer Pattern: (input_ids) / / Shape / | / Gather (indices=1) / | / Add (optional, B=0) / | Gather (segment_ids) Unsqueeze (axes=0) \ | | \ Gather Slice (data[1,512], starts=0, ends=*, axes=1, steps=1) \ / | Add Gather \ / Add | LayerNormalization ÚSlicerDrr5NFrééé Zaxesr2rr6) r r Úget_constant_valuer9r;rMrrHZget_opset_versionrZcheck_node_attributeZ get_parentr+) rrIrJrKÚpathÚsliceZ unsqueezeZ slice_weightZ opset_versionr?ZgatherrMrrrÚmatch_position_embedding_bertñsTüÿþ  ÿ z4FusionEmbedLayerNoMask.match_position_embedding_bertcCs(| |||¡r dS| |||¡rdSdS)NTF)rYrOrPrrrÚmatch_position_embedding9s z/FusionEmbedLayerNoMask.match_position_embeddingc Cs®|jd}|r |jdnd}|jd}|js!|jjdd|_d|_|jdurs|j |¡}|j |¡}|r6|s8J‚t|ƒdkrLt|ƒdkrL|d|dksYt d|›d|›¡dS|rs|j  ||¡sst d |›d |j |¡›¡dS|j  |jd ¡} | dus‡t| j ƒdkrŽt d ¡dS|j  |jd ¡} | dus¬t| j ƒdks¬| j d| j dkr³t d ¡dS|rÚ|j  |jd ¡} | dusÓt| j ƒdksÓ| j d| j dkrÚt d¡dS| j d | j d krt  d|jd ›d| j d ›d|jd ›d| j d ›¡|rU| j d | j d kr-t  d|jd ›d| j d ›d|jd ›d| j d ›¡| j d | j d krUt  d|jd ›d| j d ›d|jd ›d| j d ›¡dS)zXSanity check of embedding weights, and match hidden_size of weights and shape of inputs.rNT)Úupdater5z^Cannot fuse EmbedLayerNormalization: input_ids and position_ids not matched in 2nd dimension: z vs FzYCannot fuse EmbedLayerNormalization: input_ids and segment_ids does not have same shape: z != rzICannot fuse EmbedLayerNormalization: word embedding table is not expectedzMCannot fuse EmbedLayerNormalization: position embedding table is not expectedzLCannot fuse EmbedLayerNormalization: segment embedding table is not expectedzword_embedding_table (z) size z <= position_embedding_table (z <= segment_embedding_table (zposition_embedding_table () r9rr Zinfer_runtime_shaperZget_edge_shaper;r<ÚinfoZ compare_shaperVrMÚwarning) rÚword_embedding_gatherÚsegment_embedding_gatherrIrJÚ segment_idsÚ position_idsZinput_ids_shapeZposition_ids_shapeZword_embedding_tableZposition_embedding_tableZsegment_embedding_tablerrrÚcheck_embeddingGsj        ÿÿ   2ÿ2ÿ2ÿz&FusionEmbedLayerNoMask.check_embeddingÚ input_namecCsdd}|j |¡}|dur&|jjjtjkr |j |¡\}}||fS|}||fS|j |¡\}}||fS)a¨Cast a graph input or node input to int32. Args: input_name (str): name of graph input or node input Returns: A tuple of casted input name and the cast node. int32_output (str): If input is int32, it is the input name, Otherwise it is output name of Cast node. input_cast_node (Union[None, NodeProto]): Cast node. It could be None if input is int32. N) r Úfind_graph_inputÚtypeZ tensor_typeZ elem_typerZINT32rZcast_input_to_int32)rrcZinput_cast_nodeZ graph_inputZ int32_outputrrrÚ cast_to_int32‘s üþz$FusionEmbedLayerNoMask.cast_to_int32FrJr^rIr_rac Csªg} | |¡\}} |j d¡} |jdkr|jd} |jd} n |jd} |jd} d}|durL| |jd¡\}} |||jd|jd|jd| | g}n|d|jd|jdd| | g}|durp| d¡| |¡\}} | |¡| d | d g}|r‰|dur€|n| d }| |¡tjd||| d }d |_|j D]}|j dkr¦|j   |g¡q˜t |j ƒdkr¹|j   t  dd¡g¡|  |¡| D] }|j|j|j <qÀ|j  | ¡||_|S)agCreate an EmbedLayerNormalization node. Note that segment embedding is optional. Args: input_ids (str): input_ids for word embeddings layernorm (NodeProto): LayerNormalization or SkipLayerNormalization node. word_embedding_gather (NodeProto): the Gather node for word embedding position_embedding_gather (NodeProto): the Gather node for position embedding segment_embedding_gather (Union[None, NodeProto]): the Gather node for segment embedding, or None. Returns: NodeProto: the EmbedLayerNormalization node created. r rrr5rSNrÚÚ_outputZ_dummy_mask_indexZ_embedding_sum)ZoutputsÚnamez com.microsoftÚepsilongê-™—q=)rfr Zcreate_node_namer+r9ÚappendrZ make_nodeÚdomainÚ attributeriÚextendr;Zmake_attributeZthis_graph_nameZnode_name_to_graph_nameÚ nodes_to_addr)rrJr$r^rIr_raÚembedding_sum_outputÚembedding_sum_namerorNZ node_nameÚgammaÚbetaZembed_node_inputsr`Zembed_node_outputsrirZattr?rrrÚcreate_fused_node¨sl      ù ù   ü  €  z(FusionEmbedLayerNoMask.create_fused_nodecCs$|j |jd|jd¡d|_dS)NrT)r Úreplace_input_of_all_nodesr7Z prune_graph)rr$rrrrÚ finish_fusion s z$FusionEmbedLayerNoMask.finish_fusioncCs*|jdkot|jƒdkot|jdƒdkS)NrrSr)r+r;r7)rr?rrrÚ"is_skip_layer_norm_with_sum_outputs*z9FusionEmbedLayerNoMask.is_skip_layer_norm_with_sum_outputc Csx| |¡}|dur dS|\}}|jd} |jd} |j||dds#dS| |d|¡s,dS|jdkrP| |¡} d} |} | rA|jdnd}|duoN|j |¡du}n@|} |jdkrYdnd} t |jƒ| krg|j| nd}|duot|j |¡du}|o‚||vo‚t ||ƒdk}|duo|jdkp|p|} |j | ||||| | |r|ndd}| r´d | j| <|s´|j  ||jd ¡|  ||¡d S) NFr©r&rrSr2r)rprqZ_no_use__to_be_removed_r5T) r#r9rBrbr+rwr7r Zfind_graph_outputr;rtrurv)rr$Úadd_before_layernormr%rKÚoptional_segment_gatherÚ two_gatherr^rIrJraZneed_embedding_sum_outputZsum_output_indexZnode_with_sum_outputZ sum_outputZis_sum_graph_outputZis_sum_used_by_multiple_nodesrrrrÚ fuse_gpt2sX      ÿýÿÿ ø  z FusionEmbedLayerNoMask.fuse_gpt2c Cs‚| |¡}|dur dS|\}}|jd}|j||ddsdS| |||¡s'dS| |d|¡s0dS| ||||d¡} | || ¡dS)aÄFuse embedding layer for DistilBert Args: layernorm (NodeProto): node of LayerNormalization or SkipLayerNormalization add_before_layernorm (NodeProto): the Add node before LayerNormalization, or the SkipLayerNormalization itself input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes output_name_to_node (Dict[str, List[NodeProto]]): map from output name to nodes NFrTrx)r#r9rBrZrbrtrv) rr$ryr%rKr{r^rIrJrrrrÚfuse_distilbertcs   ÿ z&FusionEmbedLayerNoMask.fuse_distilbertcCsè|j |dgdg¡}|durdS| |d¡}|durdS|\}}|jd} |j||dds0dS|j |dgdg¡} | dur@dS| d} | | | |¡sZ| || |¡sTdS|} | }| } | ||| ¡scdS| | ||| |¡} | || ¡dS) a¾Fuse embedding layer for Bert Args: layernorm (NodeProto): node of LayerNormalization or SkipLayerNormalization add_before_layernorm (NodeProto): the Add node before LayerNormalization, or the SkipLayerNormalization itself input_name_to_nodes (Dict[str, List[NodeProto]]): map from input name to nodes output_name_to_node (Dict[str, List[NodeProto]]): map from output name to nodes r2rNFrrxrT) r r r#r9rBrZrbrtrv)rr$ryr%rKZ add_2_gatherr{r^r_rJZposition_embedding_pathrIÚtemprrrrÚ fuse_bertŒs>  û z FusionEmbedLayerNoMask.fuse_bertc Cs|j |dgdg¡}|jdkr|durdS|d}d}nP|j |dgdg¡}|j |dgdg¡}|durG|durG|dur>dS|d}|d}n%|durh|durh|j |dgdg¡}|dur_dS|d}|d}n|}d}| |||||¡rwdS| ||||¡rdS| ||||¡r‹dSdS)Nr2rrrr)r r r+r|r}r) rr?r%rKZfirst_add_pathryrzr!r"rrrÚfuse¾s<    ÿÿzFusionEmbedLayerNoMask.fuse)r )NFN)N)Ú__name__Ú __module__Ú __qualname__Ú__doc__rÚstrrrÚtupler#ÚdictÚlistÚboolrBrOrQrYrZrbrfrtrvrwr|r}rr€Ú __classcell__rrrrr sT þýü ûT>/HJ÷þýüûú ùb ÿQ)2r cs8eZdZd def‡fdd„ Zdd„Z‡fdd„Z‡ZS) ÚFusionEmbedLayerNormalizationFr cstƒ |d¡||_dS)Nz with mask)rrÚuse_mask_index)rr rŒrrrräs z&FusionEmbedLayerNormalization.__init__cCsÆ|j}t|jƒdkr|j |¡t d|j¡n"t|jƒdkr1|jds1||jd<t d|j¡n t d|j¡dS|D]$}t d|j¡|jdkrS|jd|jd<q<|jd kr`|jd|jd <q.z"EmbedLayerNormalization(with mask)zHEmbedLayerNormalization will not have mask since %s is not a node output)Ú ReduceSumrFcSr‘r’r*r“rrrr/$r”r•r)rr:rrr€rŒr<r=Zincrease_counterr9r rdrr+r;Znodes_to_removerk)rr?r%rKrŽZchildren_nodesrrrrr€ýsJ                ùz"FusionEmbedLayerNormalization.fuse)F)rr‚rƒrrrr€rŠrrrrr‹ãsr‹N)ÚloggingrZ fusion_baserZ fusion_utilsrZonnxrrrZ onnx_modelrrr<r r‹rrrrÚs    X