o
    iI                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ eeZdd Zdd Zd	d
 Zdd Zdd ZedkrVe  dS dS )    N)setup_logger)BenchmarkRecord)WhisperConfigWhisperProcessorc                  C   s  t  } | jddtddd | jddd dd	 | jd
dd ddgdd | jddtdd | jddtdd | jddddd | jddddd | jdtdd  | jd!td"d  | jd#tdd$d | jd%tdg d&d'd( | jd)tdg d*d+d( | jd,td-d.d/ | jd0ddd1d | jd2tdd3d/ | jd4td d5d/ | jd6ddd7 |  }t|d8|jd9d: 	d;d< d=|j
 d<|j }|js||_tj|jdd> | jd?9  _|S )@Nz-a--audio-pathTz0Path to folder of audio files for E2E evaluation)typerequiredhelpz-lz
--languagezLanguage of audio file)defaultr	   z-tz--taskZ
transcribe	translatezTask to complete)r
   choicesr	   z-w--warmup-runs   )r   r
   z-n
--num-runs
   z--hf-pt-eagerF
store_truez,Benchmark in PyTorch without `torch.compile`)r
   actionr	   z--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`--hf-ort-dir-pathzDPath to folder containing ONNX models for Optimum + ORT benchmarking)r   r	   --ort-model-pathz'Path to ONNX model for ORT benchmarking--model-namez9Model name in Hugging Face (e.g. openai/whisper-large-v2)--precision)Zint8Zfp16Zfp32zPrecision to run model)r   r   r   r	   --device)cpucudaZrocmzDevice to benchmark models--device-idr   zGPU device ID)r   r
   r	   z	--verbosezPrint detailed logsz	--timeoutz8Number of mins to attempt the benchmark before moving on--log-folderz'Path to folder to save logs and results--tune)r
   r   
model_size/.-z./)exist_ok<   )argparseArgumentParseradd_argumentstrint
parse_argssetattr
model_namesplitreplacer   	precision
log_folderosmakedirstimeout)parserargsZlog_folder_name r5   o/home/kim/smarthome/.venv/lib/python3.10/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.pyget_args   s    r7   c                  C   s  g }d }d}d}d}d}d\}	}
d\}}d\}}}}d\}}d}d}d	}d
}t |}|D ]}|dd}||v r>d}n||v rEd}n
||v sM||v rOd}||v rat|t||d }q.||v rt|t||d }|dkr}||}	}
d }|dkr||}}d }q.||v rt|t||d }|| }|d }q.||v r4d|v rt||dd |d d }n |||t| d  dd}t||  d }t|d }|	r|	nd|r|nd |r|nd }|d }|r|| nd}t	
d| d t	
d| d t	
d|  |||	|
|r |nd|r&|nd|||||g
 }|| q.W d    |S 1 sAw   Y  |S )NzLoad audio: zFeature extraction: zEvaluating PyTorch...zEvaluating ONNX Runtime...)NN)NNNNz	Latency: zThroughput: zGenerated token length: zpeak=
 z
load-audiozfeature-extractionprocess   ZCPU=   z MB'"Zmax_used_MBr   r   zTotal latency: z szAudio duration: zReal-time factor: )openr-   floatlenrfindr(   findjsonloadsloggerinfoappend) 	device_idlog_filebase_resultsentriesstepZload_audio_patternZfeat_ext_patternZpytorch_patternZonnxruntime_patternZload_audio_latency_sZload_audio_throughput_sZfeat_ext_latency_sZfeat_ext_throughput_stoken_lengthZ	latency_sZper_token_latency_sper_token_latency_msZ
throughputZmemoryZlatency_patternZthroughput_patternZtoken_length_patternZmemory_patternfZ
input_linelineZpeakusageZtotal_latencyZaudio_durationrtfentryr5   r5   r6   process_log_file   s   




&"




CCrW   c                 C   s  dd l }|j| g dd}|d d|d< |d d|d< |d d|d< |d	 d|d	< |d
 d|d
< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< |d d|d< dd l}|j}tdd |D }d}d}|r|d dd }|d dd }g }	| D ]\}
}|d dkrt|d |d |d |d ||}nt|d |d |d |d t	j
t	j}|d |jjd< |d |j_|d |j_|d |jjd< |d	 |jjd < |d
 |jjd!< |d |jjd"< |d |jjd#< |d |jjd$< |d |jjd%< |d |jjd&< |d d' |j_|d |j_|d |j_|	| qt||	 t|d(d)|	 td*| d+ d S ),Nr   )Warmup RunsMeasured Runs
Model NameEngine	PrecisionDevice
Audio FileDuration (s)Token LengthLoad Audio Latency (s)Load Audio Throughput (qps)Feature Extractor Latency (s)"Feature Extractor Throughput (qps)Latency (s)Per Token Latency (ms/token)Throughput (qps)Memory (GB)Real Time Factor (RTF))columnsrX   r(   rY   r_   rB   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   c                 S   s(   g | ]}|j d v r|j  d|j qS ))onnxruntimezonnxruntime-gpu==)keyversion).0ir5   r5   r6   
<listcomp>&  s   ( z save_results.<locals>.<listcomp>r9   rl   r>   r[   rk   rZ   r\   r]   r^   
audio_filedurationrP   Zload_audio_latencyZload_audio_throughputZfeature_extractor_latency_sZ feature_extractor_throughput_qpsrQ   rU   r<   .csvz.jsonzResults saved in !)ZpandasZ	DataFrameZastypepkg_resourcesworking_setsortedr,   Ziterrowsr   torch__name____version__configZ
customizedwarmup_runsZmeasured_runsmetricsZlatency_ms_meanZthroughput_qpsZmax_memory_usage_GBrJ   Zsave_as_csvZsave_as_jsonr-   rH   rI   )resultsfilenamepdZdfrv   Zinstalled_packagesZinstalled_packages_listZort_pkg_nameZort_pkg_versionrecords_rowrecordr5   r5   r6   save_results   sn    r   c              	   C   s   | dt j  dd}tj| j|}t|d'}tj|||d}z|	| j
 W n tjy8   |  Y nw W d    n1 sCw   Y  td | j| j| j|| j| j||g}	t| j||	}
|
S )Nr   %Y-%m-%d_%H:%M:%Sz.logw)stdoutstderrz Gathering data from log files...)datetimenowr0   pathjoinr/   rA   
subprocessPopenwaitr2   TimeoutExpiredkillrH   rI   r}   num_runsr+   r.   devicerW   rK   )r4   benchmark_cmdZenginerr   rs   Zlog_filenamelog_pathrL   r:   rM   r   r5   r5   r6   	benchmarkQ  s.   

r   c                  C   sr  t  } t| j t| j dtjj_	t
| j}t| j}|j| j| jd}|jgdd |D  }| jr?| jr?dt|gng }| jrM| jrMdt|gng }| jrUdgng }g }t| jD ]:}	tj| j|	}
ztj|
d}W n' ty } zd}tjd	| dd
 td|	 d W Y d }~nd }~ww td|
 d | jrdddd|
ddd| jd| jd| jdt| j dt| j!dt| j"d| j#g| }td t	| |d|	|}|$| | j%rdddd|
ddd| jd| jd| jdt| j dt| j!dt| j"d| j#g| }td t	| |d |	|}|$| | j&rYdddd|
dd!d"| j&d| jd| jd| jdt| j dt| j!dt| j"d| j#g| }td# t	| |d$|	|}|$| | j'rdddd|
dd%d&| j'd| jd| jd| jdt| j dt| j!dt| j"d| j#g| | }td' t	| |d(|	|}|$| q_| j( d)| j d*t)j)* d+d,}t+|tj| j#| d S )-NT)languagetaskc                 S   s   g | ]}|d  qS )r>   r5   )ro   Ztoken_idr5   r5   r6   rq   w  s    zmain.<locals>.<listcomp>z--decoder-input-idsr   )r   r   z@An error occurred while trying to calculate the audio duration: )exc_infozJIf you get an error that says:
	soundfile.LibsndfileError: Error opening 'zt': File contains data in an unknown format.
you may not have installed `ffmpeg` in addition to installing `librosa`.zTesting z...pythonz-mzmodels.whisper.benchmarkr   z--benchmark-typezhf-pt-eagerr   r   r   r   r   r   r   z'Benchmark PyTorch without torch.compilezpytorch-eagerzhf-pt-compilez$Benchmark PyTorch with torch.compilezpytorch-compilezhf-ortr   z Benchmark Optimum + ONNX Runtimezoptimum-ortZortr   zBenchmark ONNX Runtimerk   r!   r   r   rt   ),r7   r   verboserH   rI   __dict__ry   backendsZcudnnr   r   Zfrom_pretrainedr+   r   Zget_decoder_prompt_idsr   r   Zdecoder_start_token_idr'   Ztuner0   listdir
audio_pathr   r   librosaZget_duration	ExceptionwarningZhf_pt_eagerr.   r   rK   r}   r   r/   extendZhf_pt_compileZhf_ort_dir_pathZort_model_pathr   r   r   r   )r4   r|   	processorZhf_forced_decoder_idsZort_forced_decoder_idsZhf_decoder_input_ids_cmdZort_decoder_input_ids_cmdZort_tune_cmdZall_resultsrr   r   rs   er   r   Zcsv_filer5   r5   r6   mainl  s*  










"r   __main__)r$   r   rF   loggingr0   r   r   ry   Zbenchmark_helperr   r~   r   Ztransformersr   r   	getLoggerrz   rH   r7   rW   r   r   r   r5   r5   r5   r6   <module>   s,   
 [Z "
