o
    ¢Äiì5  ã                   @   s\   d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	m
Z
 eeƒZG dd„ deƒZdS )	é    )Ú	getLogger)ÚAttentionMaskÚFusionAttention)ÚAttentionMaskFormat)Ú	NodeProto)Ú	OnnxModelc                       sP   e Zd ZdZdededef‡ fdd„Zdedeeef fd	d
„Z	dd„ Z
‡  ZS )ÚFusionAttentionClipzB
    Fuse Attention subgraph of Clip into one Attention node.
    ÚmodelÚhidden_sizeÚ	num_headsc                    s.   t |ƒ}tj|_tƒ j||||ddgd d S )NFÚSkipLayerNormalization)Zuse_multi_head_attentionZsearch_op_types)r   r   ZNoMaskZmask_formatÚsuperÚ__init__)Úselfr	   r
   r   Úattention_mask©Ú	__class__© úh/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_attention_clip.pyr      s   
úzFusionAttentionClip.__init__Ú	reshape_qÚreturnc                 C   sZ  | j  |dd¡}|du st|jƒdkr| j| jfS | j  |jd ¡}|du r,| j| jfS t|ƒdks8|d dkr>| j| jfS |d }| j  |jd ¡}|du rU| j| jfS t|ƒdksa|d dkrg| j| jfS |d }|| }| jdkrŒ|| jkrŒ| jrŒt 	d| j› d	|› d
¡ d| _| jdkr©|| jkr©| j
r©t 	d| j› d	|› d
¡ d| _
||fS )zÏDetect num_heads and hidden_size for ONNX model from MiDaS
        Args:
            reshape_q (NodeProto): reshape node for q
        Returns:
            Tuple[int, int]: num_heads and hidden_size
        ÚConcaté   Né   é   r   é   z--num_heads is z. Detected value is z. Using detected value.Fz--hidden_size is )r	   Úmatch_parentÚlenÚinputr   r
   Zget_constant_valueZnum_heads_warningÚloggerÚwarningZhidden_size_warning)r   r   ÚconcatZnum_head_valuer   Zhead_size_valueZ	head_sizer
   r   r   r   Úget_num_heads_and_hidden_size&   s6   ÿz1FusionAttentionClip.get_num_heads_and_hidden_sizec           '      C   st  d }d }dD ]}| j  |d|¡}|d ur|}|}qd }|d ur%|jd }nDdD ];}d }| j  |d|¡}	| j  |d|¡}
|	d urB|	}n|
d urH|
}|d u rMq'| j  |d|d¡}|d u r[q'|jd }|} |d u rid S | j  |g d¢d	| d d dddg¡}|d u r–| j  |g d
¢g d¢¡}|d u r–t d¡ d S |d |d |d }}}| j  |g d¢g d¢¡}|d u rË| j  |g d¢g d¢¡}|d u rËt d¡ d S |d |d }}d }d }g }| j j|g d¢g d¢|d}|d u rT| j  |ddgddg¡}|d u rS| j  |g d¢g d¢¡}|d ur|d	 }nS| j  |g d¢g d¢¡}|d u rS| j  |g d¢g d ¢¡}|d ur;|d }n,| j  |g d!¢g d"¢¡}|d u rSt d#¡ d S nt|ƒd	ks]J ‚d	|d  }|d }|d }| j  |g d$¢g d%¢¡}|d u r™| j  |g d¢g d&¢¡}|d u r”t d'¡ d S |d	 }n|d }|d |d }}| j  |g d(¢g d)¢¡}|d u rÏ| j  |g d¢g d¢¡}|d u rÏt d*¡ d S |d |d }}|jd |ksð|jd |ksð|jd |kr÷t d+¡ d S |  	|¡\}} |dks| dkrt d,¡ d S |}!d-}"d }#d }$|d urq|jd	 d.kr*|jd	 }"nG| j  |g d/¢g d0¢¡}%|%d urA|jd	 }"n0| j  |g d1¢|dddddg¡}#| j  |g d2¢|ddddg¡}$|#d u rq|$d u rqt d3¡ d S | j
d |||||||| ||!jd |"d |#d up‹|$d ud4}&|&d u ršt d5¡ d S | j |&¡ | j| j|&j< | j |!|g¡ |  |&j¡ d6| _d S )7N)r   r   r   r   )r   r   ÚAddZLayerNormalizationF)r#   ÚMatMulÚReshapeÚ	Transposer%   r$   r   )r#   r$   r%   r&   r$   )r   Nr   r   r   z(fuse_attention: failed to match qkv pathr   r   éÿÿÿÿ)r%   r&   r%   r#   r$   )r   r   r   r   N)r&   r%   r#   r$   )r   r   r   Nz&fuse_attention: failed to match v pathéþÿÿÿ)ÚSoftmaxr%   r#   r%   r$   )r   r   r   Nr   )Zreturn_indicer)   r$   )r)   r#   ÚMulr$   )r   r   r   r   )r)   r*   r$   )r   r   r   )ÚCastr+   r)   r#   r*   r$   )r   r   r   r   r   r   )r+   r+   r)   r*   r$   )r   r   r   r   r   z'fuse_attention: failed to match qk path)r%   r&   r%   r*   r#   r$   )r   r   r   r   NN)r   r   r   Nz&fuse_attention: failed to match q path)r&   r%   r&   r%   r#   r$   )r   r   r   r   r   Nz&fuse_attention: failed to match k pathz>fuse_attention: expect to have same input to q, k and v matmulz9fuse_attention: failed to detect num_heads or hidden_sizeÚ r   )	ÚWhereÚSubr+   ÚExpandÚ	Unsqueezer0   r%   r%   r+   )	r   r   r   r   r   r   r   r   r   )r   r/   r0   r0   r-   ÚLess)r/   r0   r0   r-   r1   z4fuse_attention: failed to match causal mask subgraph)Z
mask_indexZq_matmulZk_matmulZv_matmulZq_addZk_addZv_addr   r
   Zfirst_inputÚoutputZ
add_qk_strÚscaleZcausalz+fuse_attention: failed to create fused nodeT)r	   r   r2   Zfind_first_child_by_typeZmatch_parent_pathr   Údebugr   r   r"   Zcreate_attention_nodeZnodes_to_addÚappendZthis_graph_nameZnode_name_to_graph_nameÚnameZnodes_to_removeÚextendZincrease_counterZop_typeZprune_graph)'r   Znormalize_nodeZinput_name_to_nodesZoutput_name_to_nodeZskip_input_indexZnode_before_layer_normÚiÚparentZ
root_inputZnode_before_layer_norm_1Znode_before_layer_norm_2ÚchildZ	qkv_nodesZreshape_qkvZtranspose_qkvZ
matmul_qkvZv_nodesZadd_vZmatmul_vZcausal_mask_input_indexZadd_maskZadd_mask_indicesZqk_nodesZ	matmul_qkZq_nodesr   Zadd_qZmatmul_qZk_nodesZadd_kZmatmul_kr   r
   Zattention_last_nodeZadd_qkZcausal_mask_nodes_1Zcausal_mask_nodes_2Zadd_qk_nodesÚnew_noder   r   r   ÚfuseT   s€  €

ü
ýý

ýýÿ
ü
ý



ý

ý

€ý
ÿ


ý
ÿ

0


ó
ýý
ò


zFusionAttentionClip.fuse)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r   Útupler"   r<   Ú__classcell__r   r   r   r   r      s    þýü.r   N)Úloggingr   Zfusion_attentionr   r   Zfusion_optionsr   Zonnxr   Z
onnx_modelr   r=   r   r   r   r   r   r   Ú<module>   s   