o
    YiP                     @   s  U d Z ddlmZ ddlmZ ddlZddd	Zejejd
ddgej	ej
ddddZejdejdejdddddejdejdejdddddejdejdejddejdddddejd ejdejddejd!d"d#d$dd%dejd&ejdejddejd!d"ejjd'd(dd%dejd)ejdejddejdddd*dejd+ejdejddejd!d"d,d-ejd.dd/dddejd0ejdejd1d2dd3dejd4ejdejd1d2ejd!d"d,d-ejd5d2d6d7id8dd3dejd9ejd:ejd;dejd<d=d>ejd?dd@edAdd%dejdBejd:ejd;dejd>ejd?ddCdDedEddFdejdGejd:ejd;dejd>ejd?ddHdIdCdDedJddFdejdKejd:ejd;dejd<d=d>ejd?ddCdDedLddFdejdMejdejddejejdNdOdPddFdgZeej edQ< ejee dRedSZdTZejjdUdVdWdXdY ZejjdUdVdWdZd[ ZejjdUdVdWd\d] Z ejjdUdVdWd^d_ Z!ejjdUdVdWejj"d`da Z#dS )bzTests for tunings.tune().   )types   )pytest_helper    NZv1beta1z?https://us-central1-autopush-aiplatform.sandbox.googleapis.com/)api_versionbase_urlbleuztest prompt template)nameprompt_templategs://sararob_test/Zoutput_uri_prefixZgcs_destination)metricsoutput_configZtest_dataset_gcs_urigemini-2.5-flashzVgs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonlZgcs_uri)
base_modeltraining_datasetz1gcs_uri parameter is not supported in Gemini API.)r	   
parametersZexception_if_mldevZtest_tune_pretuned_modelzFprojects/801452371447/locations/us-central1/models/9030969596621881344Vgs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonlzis not supported in Gemini APIZ+test_tune_pretuned_model_with_checkpoint_id3)Zpre_tuned_model_checkpoint_idr   r   configZtest_tune_simple_dpozModel display name   PREFERENCE_TUNING)tuned_model_display_nameepoch_countmethodz)parameter is not supported in Gemini API.Ztest_tune_dpo_with_beta      ?)r   r   r   betaZ*test_non_pretuned_model_with_checkpoint_idzis not supported in Gemini API.Z#test_dataset_gcs_uri_all_parameters      ?ADAPTER_SIZE_ONEz[gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_validation_data.jsonl)r   r   learning_rate_multiplieradapter_sizevalidation_datasetZ$test_dataset_vertex_dataset_resourcezHprojects/801452371447/locations/us-central1/datasets/5779918772206829568)Zvertex_dataset_resourcezAvertex_dataset_resource parameter is not supported in Gemini API.Z,test_dataset_dataset_resource_all_parameterszHprojects/801452371447/locations/us-central1/datasets/1168232753779441664ZtestlabelkeyZtestlabelvalue)r   r   r"   r#   r$   labelsZtest_tune_distillationz#meta/llama3_1@llama-3.1-8b-instructzEgs://nathreya-oss-tuning-sdk-test/distillation-openai-opposites.jsonlZDISTILLATIONzdeepseek-ai/deepseek-v3.1-maas   zIgs://nathreya-oss-tuning-sdk-test/distillation-val-openai-opposites.jsonlzDgs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test-folder)r   base_teacher_modelr   r$   
output_urihttp_optionsZtest_tune_oss_sftzigs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test-folder/postprocess/node-0/checkpoints/finalz=gs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test)r   r$   custom_base_modelr(   r)   znot supported in Gemini APIZtest_tune_oss_sft_hyperparamsgMb0?ZTUNING_MODE_FULL)r   r$   Zlearning_rateZtuning_moder*   r(   r)   Ztest_tune_oss_distillation)r   r'   r   r$   r*   r(   r)   Ztest_tune_encryption_speczZprojects/vertex-sdk-dev/locations/us-central1/keyRings/ayush-test-key/cryptoKeys/ayush-key)Zkms_key_name)Zencryption_spec
test_tableztunings.tune)fileZglobals_for_fileZtest_methodr+   )Zpytest_asynciozconfig.getoption('--private')zSkipping in pre-public tests)reasonc                 C   sd   | j jr.| jjdtjddtjddddtjddtd	d
}|j	tj
jks'J |jtks0J dS dS zTests tuning with eval config.r   r   r   tuning job with eval configr   r    r!   [gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_validation_data.jsonlr   r   r"   r#   r$   evaluation_configr   N)_api_clientvertexaituningstunegenai_typesTuningDatasetCreateTuningJobConfigTuningValidationDatasetr2   stateJobStateJOB_STATE_PENDINGclient
tuning_job rA   ^/home/kim/smarthome/.venv/lib/python3.10/site-packages/google/genai/tests/tunings/test_tune.pytest_eval_config  s&   
rC   c                 C   s   | j jrNtjtjdddddtjddtjddgtjtjd	d
dtjdddd}| jj	dtj
ddtjddddtjdd|dd}|jtjjksPJ dS dS )&Tests tuning with eval config metrics.prompt-relevanceXHow well does the response address the prompt?: PROMPT: {request}
 RESPONSE: {response}
T:You are a cat. Make all evaluations from this perspective.r	   r
   return_raw_outputZjudge_model_system_instructionr   )r	   rouge_1r   r   r   r   
test-modelZsampling_countZautorater_modelr   r   autorater_configr   r   r   r/   r    r!   r0   r1   r   N)r3   r4   r7   EvaluationConfigMetricOutputConfigGcsDestinationAutoraterConfigr5   r6   r8   r9   r:   r;   r<   r=   r?   r2   r@   rA   rA   rB   test_eval_config_with_metrics.  sL   



$rU   c                 C   s   | j jrctjtjtjddtjddddtjtjdddtjtjd	d
dgtj	tj
dddtjdddtjdddd}| jjdtjddtjddddtjdd|dd}|jtjjkseJ dS dS ) rD   rF   rG   T)rI   )Zmetric_prompt_templateZsystem_instructionZcustom_output_format_config)Zpointwise_metric_spec)use_effective_order)	bleu_specZrouge1)Z
rouge_type)Z
rouge_specr   r   r   r   rK   rL   r   i   )ZtemperatureZmax_output_tokens)r   r   rN   Zinference_generation_configr   r   r   r/   r    r!   r0   r1   r   N)r3   r4   r7   rO   ZUnifiedMetricZPointwiseMetricSpecZCustomOutputFormatConfigZBleuSpecZ	RougeSpecrQ   rR   rS   ZGenerationConfigr5   r6   r8   r9   r:   r;   r<   r=   rT   rA   rA   rB   %test_eval_config_with_unified_metrics[  sb   

%
4rX   c                 C   s   | j jrMtjdddddddiddid	d
diigtjtjdddtjdddd}| jjdtj	ddtj
ddddtjdd|dd}|jtjjksOJ dS dS )rD   rE   rF   TrG   rH   r	   r   rJ   rW   rV   r   r   r   r   rK   rL   rM   r   r   r   r/   r    r!   r0   r1   r   N)r3   r4   r7   rO   rQ   rR   rS   r5   r6   r8   r9   r:   r;   r<   r=   rT   rA   rA   rB   "test_eval_config_with_metrics_dict  sR   
)rY   c                    s`   | j jr,| jjjdtjddtjddddtjddt	d	d
I dH }|j
tjjks.J dS dS r.   )r3   r4   Zaior5   r6   r7   r8   r9   r:   r2   r;   r<   r=   r>   rA   rA   rB   test_eval_config_async  s&   
rZ   )$__doc__ r   r7   r   ZpytestZVERTEX_HTTP_OPTIONSrO   rP   rQ   rR   r2   ZTestTableItemZCreateTuningJobParametersr8   r9   ZTuningMethodr   r:   ZEncryptionSpecr+   list__annotations__setup__file__globalsZ
pytestmarkZpytest_pluginsmarkZskipifrC   rU   rX   rY   ZasynciorZ   rA   rA   rA   rB   <module>   s$  

		
  b

)
9
/