o i%a@sddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl ZddlZddlmZddlmZmZddlmZmZmZmZddlZeeZddd Zd d Z d d Z!ddZ"ddZ#ddZ$ddZ%edkre%dSdS)) annotationsN) setup_logger)add_io_bindings_as_tensorsget_initial_inputs_and_outputs) AutoConfigAutoModelForCausalLM AutoTokenizerBitsAndBytesConfigargsargparse.Namespacec Cs|jdvrd}|jdkr9|jdkr9tdddtjd}tj|jdkr$|jn|j |j |j |j |j dd||jd id }n[z&tj|jdkrD|jn|j |j |j |j |j d|jdkrVdnd d |j}Wn4ty}z(td |tj|jdkrv|jn|j |j |j |j |j ddd |j}WYd}~nd}~ww||jdkrt|}|St}|jdkrdd|jifnd}tj|j||gd}|S)N pt-compilept-eagerint4cudaTZnf4)Z load_in_4bitZbnb_4bit_use_double_quantZbnb_4bit_quant_typeZbnb_4bit_compute_dtypeZflash_attention_2Z80GB) cache_dir torch_dtypeuse_auth_tokentrust_remote_code use_cacheattn_implementationZquantization_configZ max_memoryZsdpa)rrrrrrz&Try to load a model using eager mode: eagerr CUDAExecutionProvider device_idZCPUExecutionProvider) sess_options providers)benchmark_typeonnx_precisiondevicer torchfloat16rfrom_pretrained hf_dir_path model_namerrauthtrustrto target_device ExceptionprintevalcompileortZSessionOptionsZInferenceSessiononnx_model_path)r modelZ bnb_configerepr2m/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.py get_model8st       r4c Cs"|jdkr t|di|}Wdn1swYd}|jdvr4|jdkr3tj|jnt||||j|j }| t }t |D]8}|jdvryt|di|}|jdkritj|jWdn1sswYqJ|||qJt }|||} | |fS)Nr r cpur2)rr Zno_gradrrZ synchronizer(ruse_fp16use_buffer_shareZsynchronize_inputstime perf_counterrangeZrun_with_iobindingZsynchronize_outputs) r r/runsinputsoutputsZ io_bindingstart_endZavgr2r2r3 run_inferencexs4           rAc CsFtt|||||j|j|j|j\}}t|||j||\}}||fSN) clear_cacherr(r6r7enginerAZ warmup_runs) r r/config tokenizer prompt_lengthpromptr<r=r?r2r2r3prepare_model_for_inferences rIcCsttjdSrB)gcZcollectr rZ empty_cacher2r2r2r3rCsrCcCsvtj|ddddddddd |d d d |d d d |d d |d ddgd}|j|ddtd|ddS)Nz Batch Sizez Prompt LengthzPrompt Processing Latency (ms)z"Prompt Processing Throughput (tps)zSampling Latency (ms)zSampling Throughput (tps)z"First Token Generated Latency (ms)z&First Token Generated Throughput (tps)Average Latency of First z Tokens Generated (ms)Average Throughput of First z Tokens Generated (tps)zWall-Clock Latency (s)zWall-Clock Throughput (tps))columnsF)indexzResults saved in !)pdZ DataFrameZto_csvloggerinfo)resultsfilenameZ gen_lengthZdfr2r2r3 save_resultss(  rVc Cst}|jddtdgdd|jddtdd d |jd d dd dd|jdddd dd|jddttjdddd|jdtddd|jddddd|jd d!dtjdd"d#d$d%d&|jd'dd d(d|jd)dd d*df|jd+d,d-d.|jd/d0d1d.|jd2d3dtd4gd5d6d7|jd8d9td:d;d|jdnd?d?d>gd@|jdAdBtdCdD|jdEdFtdGdD|jdHdItdJdD|jdKtdLdD| }t j |jt|jdM|jvrt|dN|jdO|jdPkr|jdQ|jif|_|jdMkr|jsJdR|jdS|_|jdS|_t|dT|j|jdUvs8|jdVkr:|jd?kr:d4ndW|_|jd?krIdX|jn|j}|jdWkrUtjntj}|jdMkr`dMndY}t|dZ|t|d[|t|d\|t|d]|jdWk|jo|dMk|_|S)^Nz-btz--benchmark-typeT)rr r-)typerequiredchoicesz-mz --model-nameFzrbrarrczcuda:ptr(rrDr6)argparseArgumentParser add_argumentstrospathjoinintr rZ is_available parse_argsnprandomseedZ manual_seedrsetattrrupperrgrr. batch_sizessplitprompt_lengths precisionr!Zfloat32r7)parserr r(rrDr2r2r3get_argss       *   r}c9st}tdt|jd}t|j}tj|ddd}Wdn1s(wYt j |j dkr7|j n|j |j |j|jd}tj |j dkrL|j n|j |j |j|jd}t|}g}t|j|jD]u\}}t|t|}}td|d|t||j} ||vrttd |d |jd |jd |d |d|d|d|d|jd|d||g|} ||g} ztdt|||||| \} } t|||j| | \}} |d}|||}td|dtd|||d| ||gtdtt|||||| \} } | d }|j!d}|j"}t#|dr,|j$n|j%|j&}t'j(||j)t'j*d}g}g}t+,}|| krt||d| | \}} |-|t+,}| d j!ddkr| d!.dd}|j/dd"0d|j12|d|j1}t'3| d d|4}n | d dddddf}t'j5|dd"}||B|j6k}|7||j68|dg}t+,} |-| |t'j9||gdd"}|d7}|| d<t'9| d!|:t'j;8|dgd| d!<d#| vrt'j<| d#dd"d$8|dd| d#<| d j!ddkr| d ddddddf=| d <| d >|j?d%kr,| d&| d&<nh|j@stA|jBD]}!| d'|!d(| d)|!d(<| d'|!d*| d)|!d*<q5| d!j!d}"tA|jBD]2}!t'j(|||"||j)|jCd}#t'j(|||"||j)|jCd}$| Dd'|!d(|#=d'|!d*|$=iqa|| ksIt+,}%|Ed$|jFrd+tG|tH|}&tItJfd,d|}tH|}'td-|&|'d.d/dd0t.|tH|}(|(d})|d|(}*td1|)dtd2|*d|d$}+|+d},|d|+}-td3|,dtd4|-d|jd5}.t.|d|.tH|d|.}/|/d}0|d|/}1td6|.d7|0dtd8|.d7|1dt.|tH|}2|2d}3|d|2}4td6|jd7|3dtd8|jd7|4d|%|}5|||j|5}6td9|5d:td;|||j|5dtd<| |)|*|,|-|0|1|3|4|5|6g |-| WqetKy}7ztd=|d|d>|7WYd}7~7qed}7~7wwd?|j?d@tLjLMdAdB}8tN||8|jdS)CNFcSsdd|DS)NcSsi|] \}}t||qSr2)rq).0kvr2r2r3 jsz*main....)items)dr2r2r3jszmain..) object_hookr)rrrzRunning batch size = z, prompt length = z2 A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this. 1) You can change one of the keys in 'z' to be z). If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length. If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length. 2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'. zMeasuring prompt processing...iz&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation...Z input_idshead_dim)rZdtypeZlogitsZattention_mask)dimZ position_idsrriZpast_key_valueszpresent.z.keyzpast_key_values.z.value cs |kSrBr2)Zacc_timeZanomaly_threshold_factorZ min_time_sr2r3r s z Filtered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rLrKz Tokens Generated: rMzWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - Z benchmark_Z_e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Or}rrRrS__dict__openZ prompts_filejsonloadrr"r#r$rr%r&rr4 itertoolsproductrxrzrqrCZgeneration_lengthNotImplementedErrortextwrapdedentrIrAZnum_runsextendcloneshapeZnum_key_value_headshasattrrZ hidden_sizeZnum_attention_headsr Zzerosr(boolr8r9appendsumZ unsqueezerepeatZ vocab_sizeviewZgatherZsqueezeZargmaxZ eos_token_idZ masked_fillZreshapecatr'Zint64max contiguousZzero_rDr7r:Znum_hidden_layersrupdatepopZanomaly_filteringminlenlistfilterr)datetimenowrV)9r Zsize_to_promptfrErFr/Zall_csv_metricsZ batch_sizerG max_lengthrHZ csv_metricsr<r=Zaccelerator_prompt_latency_sZaccelerator_prompt_latency_msZaccelerator_prompt_thrptZ all_token_idsZcurrent_lengthZ num_headsZ head_sizeZhas_eosZaccelerator_timesZsampling_timesZwall_clock_start_timeZaccelerator_time_latency_sZsampling_start_timeZprompt_end_indicesZidxsZnext_token_logitsZ next_tokensZ tokens_to_addZsampling_end_timeiZnew_sequence_lengthZ present_keyZ present_valueZwall_clock_end_timeZ orig_sizenew_sizeZavg_sampling_latency_sZavg_sampling_latency_msZavg_sampling_thrptZfirst_token_latency_sZfirst_token_latency_msZfirst_token_thrptZhalfwayZhalfway_token_latency_sZhalfway_token_latency_msZhalfway_token_thrptZall_token_latency_sZall_token_latency_msZall_token_thrptZwall_clock_latency_sZwall_clock_thrptr0rUr2rr3mainbs               &&    O         (r__main__)r r )& __future__rrjrrJrrloggingrnrr8numpyrsZpandasrQr Zbenchmark_helperrZ llama_inputsrrZ transformersrrrr Z onnxruntimer- getLogger__name__rRr4rArIrCrVr}rr2r2r2r3s<    @ %~