
    ci             	          d dl Z d dlZd dlmZmZmZmZ d dlmZ d dl	Z	d dl
mZ d dl
mZ d dl
mZ d dlmZ d dlmZ d dlZd	d
lmZ d	dlmZ d	dlmZ d	dlmZ 	 d dlmZ  ej<                  d      Z	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ#	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ$	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ%	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ&	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ'	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ(	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ)	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ*	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ+	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ,	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ-	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ.	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ/	 d*dee e!ef   e"f   dee e!ef      de e!ef   fdZ0	 d*dee e!ef   e"f   dee e!ef      de e!ef   fd Z1	 d*dee e!ef   e"f   dee e!ef      de e!ef   fd!Z2	 d*dee e!ef   e"f   dee e!ef      de e!ef   fd"Z3	 d*dee e!ef   e"f   dee e!ef      de e!ef   fd#Z4	 d*dee e!ef   e"f   dee e!ef      de e!ef   fd$Z5	 d*dee e!ef   e"f   dee e!ef      de e!ef   fd%Z6 G d& d'ejn                        Z8 G d( d)ejn                        Z9y# e$ r dZY w xY w)+    N)AnyCallableOptionalUnion)	urlencode)_api_module)_common)types)get_value_by_path)set_value_by_path   )_evals_common)_evals_utils)_transformers)LlmAgentzvertexai_genai.evalsfrom_objectparent_objectreturnc                 :   i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )Nevaluation_item_typeevaluationItemTypegcs_urigcsUridisplay_namedisplayNameconfiggetvsetvr   r   	to_objects      </tmp/pip-target-z3e9_cxr/lib/python/vertexai/_genai/evals.py)_CreateEvaluationItemParameters_to_vertexr#   -   s     !#IK012>!"567	
 K)%1Y
Dyk$BCK.)*6YkN;K)LMK($0Y
Dxj$AB    c           
      2   i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      #t        |dgt        t        | dg      |             t        | dg      t        |dgt        | dg             t        | d	g      t        |d
gt        | d	g             t        | dg      t        |dgt        | dg             |S )Nnamer   r   data_source
dataSourceevaluation_configevaluationConfiglabelsinference_configsinferenceConfigsr   )r   r   _EvaluationRunConfig_to_vertexr    s      r"   (_CreateEvaluationRunParameters_to_vertexr/   E   s7    !#IK&".Y${VH"=>K.)*6YkN;K)LMK-)5Y[=/(JKK-./; *[#6"78)	
 K($0Y
Dxj$ABK-./;Y+,d;AT@U.VWK($0Y
Dxj$ABr$   c                     i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )Nevaluation_itemsevaluationItemsr   r   r   r   r    s      r"   (_CreateEvaluationSetParameters_to_vertexr3   h   s     !#IK,-.:Y*+T+@R?S-TUK.)*6YkN;K)LMK($0Y
Dxj$ABr$   c                 V    i }t        | dg      t        |dgt        | dg             |S )Nevaluation_functionremote_custom_functionr   r    s      r"   $_CustomCodeExecutionSpec_from_vertexr7   y   sA     !#IK/01=%&456	
 r$   c                 V    i }t        | dg      t        |dgt        | dg             |S )Nr6   r5   r   r    s      r"   "_CustomCodeExecutionSpec_to_vertexr9      sA     !#IK234@"#789	
 r$   c           
         i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | d	g      t        |d
gt        | d	g             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      #t        |dgt        t        | dg      |             t        | dg      t        |dgt        | dg             t        | dg      :t        |dgt        j                  t        | dg            D cg c]  }| c}       t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S c c}w )N
bleu_input	bleuInputexact_match_inputexactMatchInputrouge_input
rougeInputpointwise_metric_inputpointwiseMetricInputpairwise_metric_inputpairwiseMetricInputtool_call_valid_inputtoolCallValidInputtool_name_match_inputtoolNameMatchInputtool_parameter_key_match_inputtoolParameterKeyMatchInputtool_parameter_kv_match_inputtoolParameterKvMatchInputrubric_based_metric_inputrubricBasedMetricInputautorater_configautoraterConfigmetricsinstancer   )r   r   !_RubricBasedMetricInput_to_vertext	t_metricsr   r   r!   items       r"   -_EvaluateInstancesRequestParameters_to_vertexrX      s    !#IK,(4YtK,'HIK-./;Y*+T+@S?T-UVK-)5Y[=/(JKK234@#$789	
 K123?"#678	
 K123?!"678	
 K123?!"678	
 K:;<H)*?@A	
 K9:;G()>?@	
 K567C%&-[#>"?@)	
 K,-.:Y*+T+@R?S-TUK)%1Kkk${YK*HIJdTJ	
 K*&2Yd;&EFK($0Y
Dxj$AB Ks   &	Ic                 (   i }t        | dg      1t        |dgt        | dg      D cg c]  }t        ||       c}       t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S c c}w )NrQ   outputConfigoutput_configrP   rO   )r   r    _EvaluationRunMetric_from_vertexrV   s       r"    _EvaluationRunConfig_from_vertexr]      s     !#IK)%1K !yk: 1yA	
 K.)*6Y)4n=M+NOK+,-9Y+,d;AR@S.TU   Bc                 (   i }t        | dg      1t        |dgt        | dg      D cg c]  }t        ||       c}       t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S c c}w )NrQ   r[   rZ   rO   rP   )r   r   _EvaluationRunMetric_to_vertexrV   s       r"   r.   r.     s     !#IK)%1K !yk: /tY?	
 K/*+7Y(${_<M*NOK,-.:Y*+T+@R?S-TUr^   c           
          i }t        | dg      t        |dgt        | dg             t        | dg      #t        |dgt        t        | dg      |             |S )NmetricmetricConfigmetric_config)r   r   _UnifiedMetric_from_vertexr    s      r"   r\   r\     sl     !#IK($0Y
Dxj$ABK.)*6&tK.9I'JIV	
 r$   c           
          i }t        | dg      t        |dgt        | dg             t        | dg      #t        |dgt        t        | dg      |             |S )Nrb   rd   rc   )r   r   _UnifiedMetric_to_vertexr    s      r"   r`   r`   -  sl     !#IK($0Y
Dxj$ABK/*+7$T+7H%I9U	
 r$   c           
         i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | d	g      t        |d	gt        | d	g             t        | d
g      t        |dgt        | d
g             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      #t        |dgt        t        | dg      |             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )Nr&   r   r   metadata
createTimecreate_timecompletionTimecompletion_timestateevaluationSetSnapshotevaluation_set_snapshoterrorr(   r'   evaluationResultsevaluation_run_resultsr*   r)   r-   r,   r+   )r   r   r]   r    s      r"   _EvaluationRun_from_vertexrt   ?  s8    !#IK&".Y${VH"=>K-)5Y(${]O*LMK*&2Yd;&EFK,(4YkL>)JKK*+,8Y*+T+@P?Q-RSK'#/Y	4gY#?@K123?&'678	
 K'#/Y	4gY#?@K,(4YkL>)JKK-./;%&234	
 K,-.: !,[#5"67	
 K,-.:Y,-tKBTAU/VWK($0Y
Dxj$ABr$   c           
      N   i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      #t        |dgt        t        | dg      |             t        | dg      t        |dgt        | dg             |S )Ncontents!predefined_rubric_generation_specpredefinedRubricGenerationSpecrubric_generation_specrubricGenerationSpecr   r   r   _RubricGenerationSpec_to_vertexr    s      r"   )_GenerateInstanceRubricsRequest_to_vertexr}   |  s     !#IK*&2Yd;&EFK=>?K-.BCD	
 K234@#$+[#;"<=y	
 K($0Y
Dxj$ABr$   c                     i }t        | dg      t        |ddgt        | dg             t        | dg      t        |dgt        | dg             |S Nr&   _urlr   r   r    s      r"   &_GetEvaluationItemParameters_to_vertexr     ^     !#IK&".Y(${VH*EFK($0Y
Dxj$ABr$   c                     i }t        | dg      t        |ddgt        | dg             t        | dg      t        |dgt        | dg             |S r   r   r    s      r"   %_GetEvaluationRunParameters_to_vertexr     r   r$   c                     i }t        | dg      t        |ddgt        | dg             t        | dg      t        |dgt        | dg             |S r   r   r    s      r"   %_GetEvaluationSetParameters_to_vertexr     r   r$   c           
          i }t        | dg      #t        |dgt        t        | dg      |             t        | dg      t        |dgt        | dg             |S )Nmetric_spec
metricSpecrR   )r   r    _RubricBasedMetricSpec_to_vertexr    s      r"   rS   rS     sl     !#IK-)5N,[=/2I	
 K*&2Yd;&EFr$   c           
         i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |ddgt        | dg             t        | dg      t        |dgt        | dg             t        | d	g      #t        |d
gt        t        | d	g      |             |S )Nmetric_prompt_templatemetricPromptTemplatejudge_autorater_configjudgeAutoraterConfiginline_rubricsrubricsrubric_group_keyrubricGroupKeyry   rz   r{   r    s      r"   r   r     s    !#IK234@#$789	
 K234@#$789	
 K*+,8y)/01	
 K,-.:Y)*D?Q>R,STK234@#$+[#;"<=y	
 r$   c                 :   i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )	Nprompt_templatepromptTemplategenerator_model_configmodel_configrubric_content_typerubricContentTyperubric_type_ontologyrubricTypeOntologyr   r    s      r"   r|   r|     s     !#IK+,-9Y)*D?P>Q,RSK234@Y(${=U<V*WXK/01=+,d;AV@W.X	
 K012>!"567	
 r$   c           
      2   i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | d	g      #t        |d
gt        t        | d	g      |             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )NbleuSpec	bleu_spec	rougeSpec
rouge_specpointwiseMetricSpecpointwise_metric_specllmBasedMetricSpecllm_based_metric_speccustomCodeExecutionSpeccustom_code_execution_specpredefinedMetricSpecpredefined_metric_speccomputationBasedMetricSpeccomputation_based_metric_spec)r   r   r7   r    s      r"   re   re     s_    !#IK*&2YtK*'FGK+'3Y[;-(HIK/01=$%456	
 K./0<$%345	
 K345A)*0[#<"=>		
 K012>%&567	
 K678D,-;<=	
 r$   c           
      2   i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | d	g      #t        |d
gt        t        | d	g      |             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r9   r    s      r"   rg   rg   N  s_    !#IK+'3Yd;&FGK,(4YtK,'HIK123?"#678	
 K123?!"678	
 K678D&'.[#?"@A9	
 K234@#$789	
 K9:;G)*>?@	
 r$   c                      e Zd Zddddededee   deej                     dej                  f
dZdddddd	d
ee   dee   dej                  dej                  deeeef      deeeej                  f      deej                     dej                  fdZddddee   dee   deej"                     dej$                  fdZddddddddddddddddeej(                     deej*                     deej,                     deej.                     deej0                     deej2                     deej4                     deej6                     deej8                     deej:                     deej>                     deeej@                        deejB                     deejD                     dejF                  fd Z$dddd!d"eejJ                     d#eejL                     d$eejN                     deejP                     dejR                  f
d%Z*dd&d
edeejV                     dej                  fd'Z,dd&d
edeejZ                     dej$                  fd(Z.dd&d
edeej^                     dej                  fd)Z0d*ejb                  dejF                  fd+Z2ddddd,d-e3ee4jj                  ejl                  f   d.ee3ee7e8ge8f   f      d/ee3eejr                  e:f      d0ee   deejv                     dejl                  fd1Z<dddd2d3e3e4jj                  ejz                  eejz                     f   deeej@                        d0ee   deej|                     d4e8dej~                  fd5Z@dd&d3ejz                  deej@                     d6edeej                     dej                  f
d7ZCdddddddd8d-e3ed9ejl                  f   d:ed;ee   d<ed=   d>ed?   d@eee      dAee3edBf      dCeeee8f      deejP                     dejl                  fdDZD eEj                  dE      dFddGd
edHeGdeejV                     dej                  fdI       ZH eEj                  dJ      dddddddKd3e3ej                  ejl                  f   d6edeej                     d
ee   dee   dLeej                  j                     deeeej                  f      deeeef      deej                     dej                  fdM       ZM eEj                  dN      dd&d
edeejZ                     dej$                  fdO       ZN eEj                  dP      dd&d
edeej^                     dej                  fdQ       ZO eEj                  dR      ddddej                  dedee   deej                     dej                  f
dS       ZQ eEj                  dT      ddddee   dee   deej"                     dej$                  fdU       ZRy)VEvalsNr   r   r   r   r   r   r   c                :   t        j                  ||||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }	|	r| dt        |	       }|j                  dd       d}
|j                  ,|j                  j                  |j                  j                  }
t        j                  |      }t        j                  |      }| j                  j                  d	|||
      }|j                   si nt#        j$                  |j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S z,
        Creates an EvaluationItem.
        r   r   r   r   6This method is only supported in the Vertex AI client.r   r2   _query?r   Npostresponsekwargs)r
   _CreateEvaluationItemParameters_api_clientvertexai
ValueErrorr#   get
format_mapr   popr   http_optionsr	   convert_to_dictencode_unserializable_typesrequestbodyjsonloadsEvaluationItem_from_response
model_dump_verify_responseselfr   r   r   r   parameter_modelrequest_dictrequest_url_dictpathquery_paramsr   r   response_dictreturn_values                 r"   _create_evaluation_itemzEvals._create_evaluation_item  sy     ??!5%	
 ((UVVD_UL+//7(334DE(#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++FD,U"*--TZZ5N++::"?+E+E+G ; 
 	)),7r$   r&   r   r+   r,   r   r&   r'   r)   r+   r,   c          	         t        j                  |||||||      }| j                  j                  st	        d      t        |      }	|	j                  d      }
|
rdj                  |
      }nd}|	j                  d      }|r| dt        |       }|	j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |	      }	t        j                  |	      }	| j                  j                  d	||	|      }|j                   si nt#        j$                  |j                         }| j                  j                  rt'        |      }t         j(                  j+                  ||j-                         
      }| j                  j/                  |       |S z+
        Creates an EvaluationRun.
        )r&   r   r'   r)   r+   r,   r   r   r   evaluationRunsr   r   r   Nr   r   )r
   _CreateEvaluationRunParametersr   r   r   r/   r   r   r   r   r   r   r	   r   r   r   r   r   r   rt   EvaluationRunr   r   r   r   r&   r   r'   r)   r+   r,   r   r   r   r   r   r   r   r   r   r   s                    r"   _create_evaluation_runzEvals._create_evaluation_run  s   "  >>%#//
 ((UVVCOTL+//7'223CD'#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++FD,U"*--TZZ5N$$6}EM**99"?+E+E+G : 
 	)),7r$   r1   c                8   t        j                  |||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}	|j                  ,|j                  j                  |j                  j                  }	t        j                  |      }t        j                  |      }| j                  j                  d	|||	      }
|
j                   si nt#        j$                  |
j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S z+
        Creates an EvaluationSet.
        r1   r   r   r   r   evaluationSetsr   r   r   Nr   r   )r
   _CreateEvaluationSetParametersr   r   r   r3   r   r   r   r   r   r   r	   r   r   r   r   r   r   EvaluationSetr   r   r   r   r1   r   r   r   r   r   r   r   r   r   r   r   s                r"   _create_evaluation_setzEvals._create_evaluation_set   sv     >>-%
 ((UVVCOTL+//7'223CD'#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++FD,U"*--TZZ5N**99"?+E+E+G : 
 	)),7r$   r;   r=   r?   rA   rC   rE   rG   rI   rK   rM   rO   rQ   rR   r   r;   r=   r?   rA   rC   rE   rG   rI   rK   rM   rO   rQ   rR   c                N   t        j                  |||||||||	|
||||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||      }|j                   si nt#        j$                  |j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S z>
        Evaluates instances based on a given metric.
        r   r   r   z:evaluateInstancesr   r   r   Nr   r   )r
   #_EvaluateInstancesRequestParametersr   r   r   rX   r   r   r   r   r   r   r	   r   r   r   r   r   r   EvaluateInstancesResponser   r   r   r   r;   r=   r?   rA   rC   rE   rG   rI   rK   rM   rO   rQ   rR   r   r   r   r   r   r   r   r   r   r   s                           r"   _evaluate_instanceszEvals._evaluate_instances7  s   4  CC!/##9"7"7"7+I*G&?-
$ ((UVVHL  ,//7+667GH+#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++FD,U"*--TZZ5N66EE"?+E+E+G F 
 	)),7r$   rw   ry   r   rv   rw   ry   c                :   t        j                  ||||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }	|	r| dt        |	       }|j                  dd       d}
|j                  ,|j                  j                  |j                  j                  }
t        j                  |      }t        j                  |      }| j                  j                  d	|||
      }|j                   si nt#        j$                  |j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S z7
        Generates rubrics for a given prompt.
        )rv   rw   ry   r   r   r   z:generateInstanceRubricsr   r   r   Nr   r   )r
   _GenerateInstanceRubricsRequestr   r   r   r}   r   r   r   r   r   r   r	   r   r   r   r   r   r   GenerateInstanceRubricsResponser   r   r   r   rv   rw   ry   r   r   r   r   r   r   r   r   r   r   s                 r"   _generate_rubricszEvals._generate_rubrics  sz     ??.O#9	
 ((UVVD_UL+//71<<=MN1#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++FD,U"*--TZZ5N<<KK"?+E+E+G L 
 	)),7r$   r   c                x   t        j                  ||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||      }	|	j                   si nt#        j$                  |	j                         }
| j                  j                  rt'        |
      }
t         j(                  j+                  |
|j-                         
      }| j                  j/                  |       |S zD
        Retrieves an EvaluationRun from the resource name.
        r&   r   r   r   zevaluationRuns/{name}r   r   r   Nr   r   )r
   _GetEvaluationRunParametersr   r   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   r   rt   r   r   r   r   r   r&   r   r   r   r   r   r   r   r   r   r   s               r"   _get_evaluation_runzEvals._get_evaluation_run  s     ;;
 ((UVV@QL+//7.99:JK.#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++E4|T"*--TZZ5N$$6}EM**99"?+E+E+G : 
 	)),7r$   c                6   t        j                  ||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||      }	|	j                   si nt#        j$                  |	j                         }
t         j&                  j)                  |
|j+                         
      }| j                  j-                  |       |S zD
        Retrieves an EvaluationSet from the resource name.
        r   r   r   zevaluationSets/{name}r   r   r   Nr   r   )r
   _GetEvaluationSetParametersr   r   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   s               r"   _get_evaluation_setzEvals._get_evaluation_set  ss     ;;
 ((UVV@QL+//7.99:JK.#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++E4|T"*--TZZ5N**99"?+E+E+G : 
 	)),7r$   c                6   t        j                  ||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||      }	|	j                   si nt#        j$                  |	j                         }
t         j&                  j)                  |
|j+                         
      }| j                  j-                  |       |S zE
        Retrieves an EvaluationItem from the resource name.
        r   r   r   zevaluationItems/{name}r   r   r   Nr   r   )r
   _GetEvaluationItemParametersr   r   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   s               r"   _get_evaluation_itemzEvals._get_evaluation_item,  ss     <<
 ((UVVA/RL+//7/::;KL/#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++E4|T"*--TZZ5N++::"?+E+E+G ; 
 	)),7r$   rd   c                    t        |t        j                        r|j                         }nt	        |      } | j
                  di |S )!Evaluates an instance of a model. 
isinstancer
   r   r   dictr   )r   rd   s     r"   evaluate_instanceszEvals.evaluate_instances^  sI     mU%N%NO)446M /M't'' 

 	
r$   )modelagentlocationr   srcr
  r  r  c                   |st        j                         }t        |t              rt         j                  j	                  |      }t        |t         j
                        r#|j                  t        d      |j                  }d}d}|r/t        |t              st        |t         j                        r|}n|}t        j                  | j                  |||||j                  |j                  ||j                  	      S )a  Runs inference on a dataset for evaluation.

        Args:
          src: The source of the dataset. Can be a string (path to a local file,
                a GCS path, or a BigQuery table), a Pandas DataFrame, or an
                EvaluationDataset object. If an Evalu
                ationDataset is provided,
                it must have `eval_dataset_df` populated.
          model: Optional type is experimental and may change in future versions.
                The model to use for inference, optional for agent evaluations.
              - For Google Gemini models, provide the model name string (e.g., "gemini-2.5-flash").
              - For third-party models via LiteLLM, use the format "provider/model_name"
                (e.g., "openai/gpt-4o"). Ensure the necessary API key (e.g., OPENAI_API_KEY)
                is set as an environment variable.
              - For custom logic, provide a callable function that accepts a prompt and
                returns a response.
          agent: This field is experimental and may change in future versions
                The agent engine used or local agent to run agent, optional for non-agent evaluations.
              - agent engine resource name in str type, with format
                `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`,
                run_inference will fetch the agent engine from the resource name.
              - Or `types.AgentEngine` object.
              - Or ADK agent in LlMAgent type.
          location: The location to use for the inference. If not specified, the
                location configured in the client will be used. If specified,
                this will override the location set in `vertexai.Client` only
                for this API call.
          config: The optional configuration for the inference run. Must be a dict or
              `types.EvalRunInferenceConfig` type.
                - dest: The destination path for storage of the inference results.
                - prompt_template: The template string to use for constructing prompts.
                - generate_content_config: The config for the Gemini generate content call.

        Returns:
          The evaluation dataset.
        N6EvaluationDataset must have eval_dataset_df populated.)	
api_clientr
  agent_enginer  r  destr   r  r   )r
   EvalRunInferenceConfigr  r  model_validateEvaluationDataseteval_dataset_dfr   strAgentEnginer   _execute_inferencer   r  r   generate_content_config)r   r  r
  r  r  r   agent_engine_instanceagent_instances           r"   run_inferencezEvals.run_inferencen  s    Z 113Ffd#11@@HFc5223""* L  %%C $%%E5;L;L)M(-%!&//''. "2211

 
	
r$   )rQ   r  r   datasetr   c          
         |st        j                         }t        |t              rt         j                  j	                  |      }t        |t
        j                        rt        j                  |      }t        |t              r?|D cg c]3  }t        |t              rt         j                  j	                  |      n|5 }}n/t        |t              rt         j                  j	                  |      }|t        j                  d      g}|rt        j                  d       t        j                  d| j                  |||j                  |j                   |d|S c c}w )aM  Evaluates candidate responses in the provided dataset(s) using the specified metrics.

        Args:
          dataset: The dataset(s) to evaluate. Can be a pandas DataFrame, a single
            `types.EvaluationDataset` or a list of `types.EvaluationDataset`.
          metrics: The list of metrics to use for evaluation.
          location: The location to use for the evaluation service. If not specified,
             the location configured in the client will be used. If specified,
             this will override the location set in `vertexai.Client` only for
             this API call.
          config: Optional configuration for the evaluation. Can be a dictionary or a
            `types.EvaluateMethodConfig` object.
            - dataset_schema: Schema to use for the dataset. If not specified, the
              dataset schema will be inferred from the dataset automatically.
            - dest: Destination path for storing evaluation results.
          **kwargs: Extra arguments to pass to evaluation, such as `agent_info`.

        Returns:
          The evaluation result.
        r  general_quality_v1)r&   zZ`kwargs` attribute in `evaluate` method is experimental and may change in future versions.)r  r  rQ   dataset_schemar  r  r  )r
   EvaluateMethodConfigr  r  r  pd	DataFramer  listMetricloggerwarningr   _execute_evaluationr   r"  r  )r   r  rQ   r  r   r   ds_items          r"   evaluatezEvals.evaluate  s<   B //1Ffd#//>>vFFgr||,--gFGgt$  '  "'40 ++::7C !G  '4(11@@I?||)=>?G NNl 00 
''!00
 
 	
)s   
8Er  c                   t        j                  || j                        }t        j                  t        j
                  |            }t        j                  ||||      }| j                  j                  st        d      t        j                  j                  |      }|j                  d      }	|	rdj                  |	      }
nd}
t        j                  j                  ||      }|j                  d      }|r|
 dt!        |       }
|j#                  d	d
       d
}|j$                  ,|j$                  j&                  |j$                  j&                  }t)        j*                  |      }t)        j,                  |      }| j                  j/                  d|
||      }|j0                  sdnt3        j4                  |j0                        }| j                  j                  rt        j                  j7                  |      }t        j8                  j;                  ||j=                               }| j                  j?                  |       |S z4Evaluates a dataset based on a set of given metrics.output_uri_prefixgcs_destination)r  rQ   r[   r   r   r   z:evaluateDatasetr   r   r   Nr    r   ) r   _resolve_metricsr   genai_typesOutputConfigGcsDestinationr
    EvaluateDatasetRequestParametersr   r   r   BatchEvaluateRequestPreparer*EvaluateDatasetRequestParameters_to_vertexr   r   prepare_metric_payloadr   r   r   r   r	   r   r   r   r   r   r   $EvaluateDatasetOperation_from_vertexEvaluateDatasetOperationr   r   r   r   r  rQ   r  r   resolved_metricsr[   r   r   r   r   r   r   r   r   r   s                   r"   batch_evaluatezEvals.batch_evaluate  s    )99'4CSCST#00'66N
  @@$'	
 ((UVV'DDooL  ,//7)445EF)#@@WW*
 $''1V1Y|456D4(48"".&&33?*11>>L..|<::<H##++FD,U"*--TZZ5N$$(EEjjM 55DD"?+E+E+G E 
 	)),7r$   )r   r   r   r   predefined_spec_namemetric_spec_parametersr   zpd.DataFramerubric_group_namer   r   z!genai_types.AutoraterConfigOrDictr   ztypes.RubricContentTyper   rA  ztypes.PrebuiltMetricrB  c       	            t        |t        j                        r$|j                  t	        d      |j                  }
nMt        |t
        t        j                  f      r"	 t        j                  | j                  |      }
nt        d      d|
j                  vrd|
j                  vrt	        d      |st	        d	      d|
j                  v rdnd}t        j                  d
t!        |
      ||       g }d}d}|r|rt        j#                  d       |rt        j#                  d       |rt        j#                  d       |rt        j#                  d       t        |t
              r|}n@t%        |d      r)	 |j'                  | j                        }|j(                  }nt        d      |st	        d      t        j*                  ||      }nq|rd|rt        j#                  d       ||||d}|j-                         D ci c]  \  }}|	|| }}}t        j.                  j1                  |      }nt	        d      |
j3                         D ]  \  }}||   }t        |t
              r-t5        j6                  t5        j8                  |      g      g}n@t        |t:              r|}n-t        j#                  d| d       |j=                  |g i       	 | j?                  ||||	      }||j@                  i}|j=                  |        |
jE                         }||d <   t        j                  d!| d       t        j                  |"      S # t        $ r}t	        d| d|       d}~ww xY w# t        $ r}t	        d|       d}~ww xY wc c}}w # t        $ rC}t        jC                  dt        |      dd |d       |j=                  |g i       Y d}~d}~ww xY w)#a
  Generates rubrics for each prompt in the source and adds them as a new column
        structured as a dictionary.

        You can generate rubrics by providing either:
          1. A `predefined_spec_name` to use a Vertex AI backend recipe.
          2. A `prompt_template` along with other configuration parameters
             (`generator_model_config`, `rubric_content_type`, `rubric_type_ontology`)
             for custom rubric generation.

        These two modes are mutually exclusive.

        Args:
            src: The source of the prompts. Can be a string (path to a local
              file, a GCS path, or a BigQuery table), a Pandas DataFrame, or
              an EvaluationDataset object. The loaded data must contain either
              a 'prompt' column (for text) or a 'request' column (for text or
              multimodal Gemini Content).
            rubric_group_name: Name for the key within the dictionary in the new
              column.
            prompt_template: Optional. Template for the rubric generation prompt. Used for
              custom rubric generation. Mutually exclusive with `predefined_spec_name`.
              If using a 'prompt' column, use {prompt} as the placeholder. If using a
              'request' column, this template is passed to the service along
              with the content.
            generator_model_config: Optional. Configuration for the model used
              in custom rubric generation. Only used if `prompt_template` is provided.
              e.g., {"autorater_model": "gemini-2.5-flash"}.
            rubric_content_type: Optional. The type of rubric content to be
              generated. Only used if `prompt_template` is provided.
            rubric_type_ontology: Optional. A pre-defined list of allowed types
              for generated rubrics. Only used if `prompt_template` is provided.
            predefined_spec_name: Optional. The name of a Predefined Metric to use
                for rubric generation (e.g., "general_quality_v1") or a types.PrebuiltMetric object.
                Mutually exclusive with `prompt_template` and its related parameters.
            metric_spec_parameters: Optional. Parameters for the Predefined Metric,
                used to customize rubric generation. Only used if `predefined_spec_name` is set.
                Example: {"guidelines": ["The response must be in Japanese."]}
            config: Optional. Configuration for the rubric generation process.

        Returns:
            An `EvaluationDataset` with an added column named `rubric_groups` in its
            `eval_dataset_df`. Each cell in this column contains a dictionary like:
            {rubric_group_name: [list[Rubric]]}.
        Nr  z+Failed to load prompt dataset from source: z	. Error: zPUnsupported type for src. Must be str, pd.DataFrame, or types.EvaluationDataset.promptr   z8Loaded dataset must have a 'prompt' or 'request' column.z"rubric_group_name cannot be empty.zBGenerating rubrics for %d prompts from column '%s', group: '%s'...zAprompt_template is ignored when predefined_spec_name is provided.zHgenerator_model_config is ignored when predefined_spec_name is provided.zErubric_content_type is ignored when predefined_spec_name is provided.zFrubric_type_ontology is ignored when predefined_spec_name is provided.resolvez"Failed to resolve PrebuiltMetric: z=predefined_spec_name must be a string or types.PrebuiltMetricz>Could not determine metric_spec_name from predefined_spec_name)metric_spec_namerB  zCmetric_spec_parameters is ignored when prompt_template is provided.)r   r   r   r   z@Either predefined_spec_name or prompt_template must be provided.textpartsz1Skipping row: Unexpected input format in column 'z'.)rv   ry   rw   r   z3Rubric generation failed for input: %s... Error: %s2   T)exc_inforubric_groupszCRubric generation complete. Added column 'rubric_groups' with key 'r   )#r  r
   r  r  r   r  r$  r%  r   _load_dataframer   	Exception	TypeErrorcolumnsr(  infolenr)  hasattrrF  r&   PredefinedMetricSpecitemsRubricGenerationSpecr  iterrowsr5  ContentPartr&  appendr   generated_rubricsrq   copy)r   r  rC  r   r   r   r   rA  rB  r   
prompts_dfeinput_columnall_rubric_groupsrubric_gen_specpredefined_specactual_predefined_spec_nameresolved_metric	spec_dictkv_row
input_datarv   r   rubric_groupprompts_with_rubricss                               r"   generate_rubricszEvals.generate_rubricsI  s+   r c5223""* L  ,,Jc2<<01*::4;K;KSQ
 b  :---):CUCU2UWXX ABB$-1C1C$CyP
O		
 BDW &^ #[ $\ .4.B+$iO&:&B&B4CSCS&TO2A2F2F/  S  / T  $88!<'=O %Y $3':(<*@	I +4//*;M$!Qq}AMIM#88GG	ROR  !))+ 	BFAs\*J*c*''{/?/?Z/P.QR J-%G~UWX "((*;R)@AB11%+:6E!	 2  !283M3MN!((6/	BB  *00A_-QRcQddfg	
 &&7KLLy   A#iPQsS h ! O$'I!%MNNO6 N>  BI
OCR(!	   "((*;R)@AABsT     M 7'M5 
NN4N	M2M--M25	N>NN	O&#8O!!O&hThe Vertex SDK GenAI evals.get_evaluation_run module is experimental, and may change in future versions.Finclude_evaluation_itemsr   rr  c                   |st        d      |j                  d      r|j                  d      d   }| j                  ||      }|r:t	        j
                  | j                  |j                  |j                        |_	        |S )aO  Retrieves an EvaluationRun from the resource name.
        Args:
          name: The resource name of the EvaluationRun. Format:
            `projects/{project}/locations/{location}/evaluationRuns/{evaluation_run}`
          include_evaluation_items: Whether to include the evaluation items in the
            response.
          config: The optional configuration for the evaluation run. Must be a dict or
              `types.GetEvaluationRunConfigOrDict` type.

        Returns:
          The evaluation run.
        Raises:
          ValueError: If the name is empty or invalid.
        name cannot be empty.	projects//r   )
r   
startswithsplitr   r   _convert_evaluation_run_resultsr   rs   r,   evaluation_item_resultsr   r&   rr  r   results        r"   get_evaluation_runzEvals.get_evaluation_run	  s    2 455??;'::c?2&D))tF)C#==$$11,, * r$   kThe Vertex SDK GenAI evals.create_evaluation_run module is experimental, and may change in future versions.r&   r   
agent_infor,   r+   r   r  c       	   	      V   |r|rt        d      t        j                  j                         }
|r<t	        |t
              r*t        j                  j                  j                  |      }
n|}
t	        |t        j                        r|j                  t        d      |
I|j                  r=|
r;|
j                  r/|j                  |
j                  k7  rt        j                  d       n|j                  |
r|
j                  |_	        t        j                  | j                  ||j                  |j                        }t        j                   |j                        }t#        j$                  t#        j&                  |            }t        j(                  || j                        }t        j*                  ||      }|
r|
j                  i }t        j,                  t        j.                  t#        j0                  t#        j2                  |
j4                        g	      |
j6                  
            ||
j                  <   |
j8                  r'|xs i }|
j8                  j;                  d      d   |d<   |sdt=        j>                          }| jA                  ||xs ||||||	      S )  Creates an EvaluationRun.

        Args:
          dataset: The dataset to evaluate. Either an EvaluationRunDataSource or an EvaluationDataset.
          dest: The GCS URI prefix to write the evaluation results to.
          metrics: The list of metrics to evaluate.
          name: The name of the evaluation run.
          display_name: The display name of the evaluation run.
          agent_info: The agent info to evaluate.
          inference_configs: The candidate to inference config map for the evaluation run.
              The key is the candidate name, and the value is the inference config.
              If provided, agent_info must be None.
              Example:
              {"candidate-1": types.EvaluationRunInferenceConfig(model="gemini-2.5-flash")}
          labels: The labels to apply to the evaluation run.
          config: The configuration for the evaluation run.

        Returns:
            The created evaluation run.
        ?At most one of agent_info or inference_configs can be provided.r  gEvaluation dataset candidate_name and agent_info.name are different. Please make sure this is intended.evaluation_setr/  r1  r[   rQ   rH  rJ  developer_instructiontoolsagent_configreasoningEngines/rw  $vertex-ai-evaluation-agent-engine-idevaluation_run_r&   r   r'   r)   r,   r+   r   )!r   r
   evals	AgentInfor  r  r  r  r  candidate_namer&   r(  r)  r   %_create_evaluation_set_from_dataframer   EvaluationRunDataSourcer5  r6  r7  _resolve_evaluation_run_metricsEvaluationRunConfigEvaluationRunInferenceConfigEvaluationRunAgentConfigrZ  r[  instructiontool_declarationsagent_resource_namery  uuiduuid4r   )r   r  r  rQ   r&   r   r  r,   r+   r   agent_info_pydanticeval_setr[   r?  r)   s                  r"   create_evaluation_runzEvals.create_evaluation_run1  s|   N +Q  6;[[5J5J5L*d+&+kk&;&;&J&J:&V#&0#gu667&&. L  #.&&'',,**.A.F.FF} ''/4G)<)A)A&$JJ  $(?(?AWAWH 338==QG#00'66N
 )HHT%%
 "55'1A
 #6#;#;#G "22!&!?!?.9.A.A + 0 06I6U6U V#/
 2CC"	 1667 #662';;AABUV =>
 $TZZ\N3D**%-// + 
 	
r$   hThe Vertex SDK GenAI evals.get_evaluation_set method is experimental, and may change in future versions.c                    |st        d      |j                  d      r|j                  d      d   }| j                  ||      S )  Retrieves an EvaluationSet from the resource name.

        Args:
          name: The resource name of the EvaluationSet. Format:
            `projects/{project}/locations/{location}/evaluationSets/{evaluation_set}`
          config: The optional configuration for the evaluation set. Must be a dict or
              `types.GetEvaluationSetConfigOrDict` type.

        Returns:
          The evaluation set.
        rt  ru  rv  rw  r   r   rx  ry  r   )r   r&   r   s      r"   get_evaluation_setzEvals.get_evaluation_set  sG    , 455??;'::c?2&D''T&'AAr$   iThe Vertex SDK GenAI evals.get_evaluation_item method is experimental, and may change in future versions.c                   |st        d      |j                  d      r|j                  d      d   }| j                  ||      }|j                  rX|j
                  t        j                  j                  k(  r1t        j                  | j                  |j                        |_        |S |j                  rV|j
                  t        j                  j                  k(  r/t        j                  | j                  |j                        |_        |S )  Retrieves an EvaluationItem from the resource name.

        Args:
          name: The resource name of the EvaluationItem. Format:
            `projects/{project}/locations/{location}/evaluationItems/{evaluation_item}`
          config: The optional configuration for the evaluation item. Must be a dict or
              `types.GetEvaluationItemConfigOrDict` type.

        Returns:
          The evaluation item.
        rt  ru  rv  rw  r   r   rx  ry  r  r   r   r
   EvaluationItemTypeRESULTr   &_convert_gcs_to_evaluation_item_resultr   evaluation_responseREQUEST'_convert_gcs_to_evaluation_item_requestevaluation_requestr   r&   r   r}  s       r"   get_evaluation_itemzEvals.get_evaluation_item  s    * 455??;'::c?2&D**V*DNN++u/G/G/N/NN DD$$fnn &  NN++u/G/G/O/OO EE$$fnn %
 r$   lThe Vertex SDK GenAI evals.create_evaluation_item module is experimental, and may change in future versions.c                ,    | j                  ||||      S )  Creates an EvaluationItem.

        Args:
          evaluation_item_type: The type of the evaluation item.
          gcs_uri: The GCS URI of the evaluation item.
          display_name: The display name of the evaluation item.
          config: The optional configuration for the evaluation item. Must be a dict or
              `types.CreateEvaluationItemConfigOrDict` type.

        Returns:
          The evaluation item.
        r   r   )r   r   r   r   r   s        r"   create_evaluation_itemzEvals.create_evaluation_item  s(    0 ++!5%	 , 
 	
r$   kThe Vertex SDK GenAI evals.create_evaluation_set module is experimental, and may change in future versions.c                *    | j                  |||      S )  Creates an EvaluationSet.

        Args:
          evaluation_items: The list of evaluation item names. Format:
            `projects/{project}/locations/{location}/evaluationItems/{evaluation_item}`
          display_name: The display name of the evaluation set.
          config: The optional configuration for the evaluation set. Must be a dict or
              `types.CreateEvaluationSetConfigOrDict` type.

        Returns:
          The evaluation set.
        r   r   )r   r1   r   r   s       r"   create_evaluation_setzEvals.create_evaluation_set
  s%    . **-% + 
 	
r$   )S__name__
__module____qualname__r  r   r
    CreateEvaluationItemConfigOrDictr   r   EvaluationRunDataSourceOrDictEvaluationRunConfigOrDictr  "EvaluationRunInferenceConfigOrDictCreateEvaluationRunConfigOrDictr   r   r&  CreateEvaluationSetConfigOrDictr   r   BleuInputOrDictExactMatchInputOrDictRougeInputOrDictPointwiseMetricInputOrDictPairwiseMetricInputOrDictToolCallValidInputOrDictToolNameMatchInputOrDict ToolParameterKeyMatchInputOrDictToolParameterKVMatchInputOrDictRubricBasedMetricInputOrDictr5  AutoraterConfigOrDictMetricOrDictEvaluationInstanceOrDictEvaluateInstancesConfigOrDictr   r   ContentOrDictPredefinedMetricSpecOrDictRubricGenerationSpecOrDictRubricGenerationConfigOrDictr   r   GetEvaluationRunConfigOrDictr   GetEvaluationSetConfigOrDictr   GetEvaluationItemConfigOrDictr  r   r	  r   r$  r%  r  r   r   r  r   EvalRunInferenceConfigOrDictr  EvaluationDatasetOrDictEvaluateMethodConfigOrDictEvaluationResultr,  EvaluateDatasetConfigOrDictr=  r@  ro  r	   experimental_warningboolr~  r  EvaluationRunMetricOrDictr  AgentInfoOrDictr  r  r  r  r  r  r  r$   r"   r   r     s 
    '+CG7 "7 	7
 sm7 ??@7 
		7x #&* ,0 BFB smB sm	B
 88B !::B c3h(B $e>>>?
B >>?B 
		BP '+BF5 s)5 sm	5
 >>?5 
		5t 7;CG8<MQKOJNJN  RVHL6:=A@D)Q U223Q $E$?$?@	Q
 e445Q !))I)I JQ  ((G(GHQ  ((F(FGQ  ((F(FGQ )122)
Q (011(
Q  $,E,N,N#O!Q" #;#D#DE#Q$ $u1123%Q& 599:'Q( <<=)Q* 
	(	(+Qr MQ?C9 {0019 ,4,,,
	9 !))I)I J9 ;;<9 
	.	.9x TX33$,U-O-O$P3			3l TX00$,U-O-O$P0			0f UY00$,U-P-P$Q0			0d
 @@
 
	(	(	
( =ACG"&?CK
 3e&=&==>K
 c8SE3J#7789	K

 c5#4#4h>?@K
 3-K
 ;;<K
 
	 	 K
j 7;"&=AF
 LL))../1
F
 $u1123F
 3-F
 99:F
 F
 
		F
\ ?CD ..D e(()	D
 D ::;D 
	'	'DV *.PTCG48MQ;??C~M 3(?(??@~M 	~M
 "#~M !))L M~M &&?@~M 'tCy1~M 'uS2H-H'IJ~M !)c3h 8~M ;;<~M 
	 	 ~M@ "W!!	- */?C" " #'	"
 ;;<" 
		"	"H "W!!	- #&*<@ +/BFj
 u44e6M6MMNj
 	j

 e556j
 smj
 smj
 U[[889j
 $e>>>?
j
 c3h(j
 >>?j
 
		j
	j
X "W!!	- @D	B B ;;<	B
 
		B	B0 "W!!	- AE	( ( <<=	(
 
		(	(T "W!!	- '+CG
 $66
 	

 sm
 ??@
 
		
	
6 "W!!	- '+BF
 s)
 sm	

 >>?
 
		
	
r$   r   c                   l	   e Zd Zddddededee   deej                     dej                  f
dZdddddd	d
ee   dee   dej                  dej                  deeeef      deeeej                  f      deej                     dej                  fdZddddee   dee   deej"                     dej$                  fdZddddddddddddddddeej(                     deej*                     deej,                     deej.                     deej0                     deej2                     deej4                     deej6                     deej8                     deej:                     deej>                     deeej@                        deejB                     deejD                     dejF                  fd Z$dddd!d"eejJ                     d#eejL                     d$eejN                     deejP                     dejR                  f
d%Z*dd&d
edeejV                     dej                  fd'Z,dd&d
edeejZ                     dej$                  fd(Z.dd&d
edeej^                     dej                  fd)Z0dd&d*ejb                  deej@                     d+edeejd                     dejf                  f
d,Z4d-ejj                  dejF                  fd.Z6 e7jp                  d/      d0dd1d
ed2e9deejV                     dej                  fd3       Z: e7jp                  d4      ddddddd5d*e;ejx                  ejz                  f   d+edeej|                     d
ee   dee   d6eej~                  j                     deeeej                  f      deeeef      deej                     dej                  fd7       ZA e7jp                  d8      dd&d
edeejZ                     dej$                  fd9       ZB e7jp                  d:      dd&d
edeej^                     dej                  fd;       ZC e7jp                  d<      ddddej                  dedee   deej                     dej                  f
d=       ZE e7jp                  d>      ddddee   dee   deej"                     dej$                  fd?       ZFy)@
AsyncEvalsNr   r   r   r   r   r   c                V  K   t        j                  ||||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }	|	r| dt        |	       }|j                  dd       d}
|j                  ,|j                  j                  |j                  j                  }
t        j                  |      }t        j                  |      }| j                  j                  d	|||
       d{   }|j                   si nt#        j$                  |j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S 7 }wr   )r
   r   r   r   r   r#   r   r   r   r   r   r   r	   r   r   async_requestr   r   r   r   r   r   r   r   s                 r"   r   z"AsyncEvals._create_evaluation_item*  s      ??!5%	
 ((UVVD_UL+//7(334DE(#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))77D,
 
 #+--TZZ5N++::"?+E+E+G ; 
 	)),7
   D'F))F'*A>F)r   r&   r'   r)   r+   r,   c          	        K   t        j                  |||||||      }| j                  j                  st	        d      t        |      }	|	j                  d      }
|
rdj                  |
      }nd}|	j                  d      }|r| dt        |       }|	j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |	      }	t        j                  |	      }	| j                  j                  d	||	|       d{   }|j                   si nt#        j$                  |j                         }| j                  j                  rt'        |      }t         j(                  j+                  ||j-                         
      }| j                  j/                  |       |S 7 wr   )r
   r   r   r   r   r/   r   r   r   r   r   r   r	   r   r   r  r   r   r   rt   r   r   r   r   r   s                    r"   r   z!AsyncEvals._create_evaluation_rune  s    "  >>%#//
 ((UVVCOTL+//7'223CD'#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))77D,
 
 #+--TZZ5N$$6}EM**99"?+E+E+G : 
 	)),7
s   D*G,G-BGr1   c                T  K   t        j                  |||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}	|j                  ,|j                  j                  |j                  j                  }	t        j                  |      }t        j                  |      }| j                  j                  d	|||	       d{   }
|
j                   si nt#        j$                  |
j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S 7 }wr   )r
   r   r   r   r   r3   r   r   r   r   r   r   r	   r   r   r  r   r   r   r   r   r   r   r   s                r"   r   z!AsyncEvals._create_evaluation_set  s      >>-%
 ((UVVCOTL+//7'223CD'#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))77D,
 
 #+--TZZ5N**99"?+E+E+G : 
 	)),7
s   D&F((F&)A>F(r   r;   r=   r?   rA   rC   rE   rG   rI   rK   rM   rO   rQ   rR   c                j  K   t        j                  |||||||||	|
||||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||       d{   }|j                   si nt#        j$                  |j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S 7 }wr   )r
   r   r   r   r   rX   r   r   r   r   r   r   r	   r   r   r  r   r   r   r   r   r   r   r   s                           r"   r   zAsyncEvals._evaluate_instances  s    4  CC!/##9"7"7"7+I*G&?-
$ ((UVVHL  ,//7+667GH+#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))77D,
 
 #+--TZZ5N66EE"?+E+E+G F 
 	)),7
s   D1F33F14A>F3r   rv   rw   ry   c                V  K   t        j                  ||||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }	|	r| dt        |	       }|j                  dd       d}
|j                  ,|j                  j                  |j                  j                  }
t        j                  |      }t        j                  |      }| j                  j                  d	|||
       d{   }|j                   si nt#        j$                  |j                         }t         j&                  j)                  ||j+                         
      }| j                  j-                  |       |S 7 }wr   )r
   r   r   r   r   r}   r   r   r   r   r   r   r	   r   r   r  r   r   r   r   r   r   r   r   s                 r"   r   zAsyncEvals._generate_rubrics9  s      ??.O#9	
 ((UVVD_UL+//71<<=MN1#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))77D,
 
 #+--TZZ5N<<KK"?+E+E+G L 
 	)),7
r  r   c                  K   t        j                  ||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||       d{   }	|	j                   si nt#        j$                  |	j                         }
| j                  j                  rt'        |
      }
t         j(                  j+                  |
|j-                         
      }| j                  j/                  |       |S 7 wr   )r
   r   r   r   r   r   r   r   r   r   r   r   r	   r   r   r  r   r   r   rt   r   r   r   r   r   s               r"   r   zAsyncEvals._get_evaluation_runv  s      ;;
 ((UVV@QL+//7.99:JK.#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))774|
 
 #+--TZZ5N$$6}EM**99"?+E+E+G : 
 	)),7
s   D%G'G(BGc                R  K   t        j                  ||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||       d{   }	|	j                   si nt#        j$                  |	j                         }
t         j&                  j)                  |
|j+                         
      }| j                  j-                  |       |S 7 }wr   )r
   r   r   r   r   r   r   r   r   r   r   r   r	   r   r   r  r   r   r   r   r   r   r   r   s               r"   r   zAsyncEvals._get_evaluation_set  s      ;;
 ((UVV@QL+//7.99:JK.#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))774|
 
 #+--TZZ5N**99"?+E+E+G : 
 	)),7
   D%F''F%(A>F'c                R  K   t        j                  ||      }| j                  j                  st	        d      t        |      }|j                  d      }|rdj                  |      }nd}|j                  d      }|r| dt        |       }|j                  dd       d}|j                  ,|j                  j                  |j                  j                  }t        j                  |      }t        j                  |      }| j                  j                  d	|||       d{   }	|	j                   si nt#        j$                  |	j                         }
t         j&                  j)                  |
|j+                         
      }| j                  j-                  |       |S 7 }wr   )r
   r  r   r   r   r   r   r   r   r   r   r   r	   r   r   r  r   r   r   r   r   r   r   r   s               r"   r  zAsyncEvals._get_evaluation_item  s      <<
 ((UVVA/RL+//7/::;KL/#''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))774|
 
 #+--TZZ5N++::"?+E+E+G ; 
 	)),7
r  r  r  c                  K   t        j                  || j                        }t        j                  t        j
                  |            }t        j                  ||||      }| j                  j                  st        d      t        j                  j                  |      }|j                  d      }	|	rdj                  |	      }
nd}
t        j                  j                  ||      }|j                  d      }|r|
 dt!        |       }
|j#                  d	d
       d
}|j$                  ,|j$                  j&                  |j$                  j&                  }t)        j*                  |      }t)        j,                  |      }| j                  j/                  d|
||       d
{   }|j0                  sdnt3        j4                  |j0                        }| j                  j                  rt        j                  j7                  |      }t        j8                  j;                  ||j=                               }| j                  j?                  |       |S 7 wr.  ) r   r4  r   r5  r6  r7  r
   r8  r   r   r   r9  r:  r   r   r;  r   r   r   r   r	   r   r   r  r   r   r   r<  r=  r   r   r   r>  s                   r"   r@  zAsyncEvals.batch_evaluate	  s     )99'4CSCST#00'66N
  @@$'	
 ((UVV'DDooL  ,//7)445EF)#@@WW*
 $''1V1Y|456D4(48"".&&33?*11>>L..|<::<H))77D,
 
 #+--TZZ5N$$(EEjjM 55DD"?+E+E+G E 
 	)),7!
s   F%I'I(B3Ird   c                   K   t        |t        j                        r|j                         }nt	        |      } | j
                  di | d{   }|S 7 w)r  Nr  r  )r   rd   r}  s      r"   r	  zAsyncEvals.evaluate_instances\	  s[      mU%N%NO)446M /M/t// 

 
 	
s   AAAArp  Frq  rr  c                8  K   |st        d      |j                  d      r|j                  d      d   }| j                  ||       d{   }|rBt	        j
                  | j                  |j                  |j                         d{   |_	        |S 7 J7 w)aP  Retrieves the EvaluationRun from the resource name.
        Args:
          name: The resource name of the EvaluationRun. Format:
            `projects/{project}/locations/{location}/evaluationRuns/{evaluation_run}`
          include_evaluation_items: Whether to include the evaluation items in the
            response.
          config: The optional configuration for the evaluation run. Must be a dict or
              `types.GetEvaluationRunConfigOrDict` type.

        Returns:
          The evaluation run.
        Raises:
          ValueError: If the name is empty or invalid.
        rt  ru  rv  rw  r   N)
r   rx  ry  r   r   %_convert_evaluation_run_results_asyncr   rs   r,   r{  r|  s        r"   r~  zAsyncEvals.get_evaluation_runn	  s     2 455??;'::c?2&D//T&/II##II$$11,,  *  Js$   A	BB>B
BBBr  r  r  c       	   	      0  K   |r|rt        d      |r9t        |t              r)t        j                  j
                  j                  |      }t        |t        j                        r|j                  t        d      |G|j                  r;|j                  r/|j                  |j                  k7  rt        j                  d       n|j                  |r|j                  |_	        t        j                  | j                  ||j                  |j                        }
t        j                   |
j                        }t#        j$                  t#        j&                  |            }t        j(                  || j                        }t        j*                  ||      }|r|j                  i }t        j,                  t        j.                  t#        j0                  t#        j2                  |j4                  	      g
      |j6                              ||j                  <   |j8                  r'|xs i }|j8                  j;                  d      d   |d<   |sdt=        j>                          }| jA                  ||xs ||||||	       d{   }|S 7 w)r  r  Nr  r  r  r/  r1  r  rH  rJ  r  r  r  rw  r  r  r  )!r   r  r  r
   r  r  r  r  r  r  r&   r(  r)  r   r  r   r  r5  r6  r7  r  r  r  r  rZ  r[  r  r  r  ry  r  r  r   )r   r  r  rQ   r&   r   r  r,   r+   r   r  r[   r?  r)   r}  s                  r"   r  z AsyncEvals.create_evaluation_run	  sZ    N +Q  *Z6..==jIJgu667&&. L  %&&OO**joo=} ''/J)3&$JJ  $(?(?AWAWH 338==QG#00'66N
 )HHT%%
 "55'1A
 *//5 "161S1S";;*5*=*=*//Z5K5KLM+ %66	2joo. --222889LMbQ => $TZZ\N3D22%-// 3 
 
 
s   JJJJr  c                   K   |st        d      |j                  d      r|j                  d      d   }| j                  ||       d{   }|S 7 w)r  rt  ru  rv  rw  r   Nr  r  s       r"   r  zAsyncEvals.get_evaluation_set	  sX     * 455??;'::c?2&D//T&/II Js   A	AAAr  c                8  K   |st        d      |j                  d      r|j                  d      d   }| j                  ||       d{   }|j                  rX|j
                  t        j                  j                  k(  r1t        j                  | j                  |j                        |_        |S |j                  rV|j
                  t        j                  j                  k(  r/t        j                  | j                  |j                        |_        |S 7 ̭w)r  rt  ru  rv  rw  r   Nr  r  s       r"   r  zAsyncEvals.get_evaluation_item
  s     * 455??;'::c?2&D00d60JJNN++u/G/G/N/NN DD$$fnn &  NN++u/G/G/O/OO EE$$fnn % ) Ks   A	DDCDr  c                L   K   | j                  ||||       d{   }|S 7 w)r  r   Nr  )r   r   r   r   r   r}  s         r"   r  z!AsyncEvals.create_evaluation_itemJ
  s:     0 33!5%	 4 
 
 
s   $"$r  c                J   K   | j                  |||       d{   }|S 7 w)r  r   Nr  )r   r1   r   r   r}  s        r"   r  z AsyncEvals.create_evaluation_setj
  s7     . 22-% 3 
 

 
s   #!#)Gr  r  r  r  r   r
   r  r   r   r  r  r  r  r  r   r   r&  r  r   r   r  r  r  r  r  r  r  r  r  r  r5  r  r  r  r  r   r   r  r  r  r  r   r   r  r   r  r   r  r  r  r  r=  r@  r   r	  r	   r  r  r~  r   r  r  r  r  r  r  r  r  r  r  r  r  r$   r"   r  r  (  s    '+CG9 "9 	9
 sm9 ??@9 
		9| #&* ,0 BFD smD sm	D
 88D !::D c3h(D $e>>>?
D >>?D 
		DT '+BF7 s)7 sm	7
 >>?7 
		7x 7;CG8<MQKOJNJN  RVHL6:=A@D)S U223S $E$?$?@	S
 e445S !))I)I JS  ((G(GHS  ((F(FGS  ((F(FGS )122)
S (011(
S  $,E,N,N#O!S" #;#D#DE#S$ $u1123%S& 599:'S( <<=)S* 
	(	(+Sv MQ?C; {001; ,4,,,
	; !))I)I J; ;;<; 
	.	.;| TX55$,U-O-O$P5			5p TX22$,U-O-O$P2			2j UY22$,U-P-P$Q2			2t ?CE ..E e(()	E
 E ::;E 
	'	'EN @@ 
	(	(	$ "W!!	- */?C# # #'	#
 ;;<# 
		#	#J "W!!	- #&*6: +/BFa u44e6M6MMNa 	a
 e556a sma sma U[[223a $e>>>?
a c3h(a >>?a 
		a	aF "W!!	- @D	  ;;<	
 
			2 "W!!	- AE	) ) <<=	)
 
		)	)V "W!!	- '+CG $66 	
 sm ??@ 
			8 "W!!	- '+BF s) sm	
 >>? 
			r$   r  )N):r   loggingtypingr   r   r   r   urllib.parser   r  google.genair   r	   r
   r5  google.genai._commonr   r   r   r   pandasr$  r3  r   r   r   rT   google.adk.agentsr   ImportError	getLoggerr(  r  r  objectr#   r/   r3   r7   r9   rX   r]   r.   r\   r`   rt   r}   r   r   r   rS   r   r|   re   rg   
BaseModuler   r  r  r$   r"   <module>r     s{  $   1 1 "  $   - : :      *
 
		1	2
 /3tCH~v-.DcN+ 
#s(^4 /3 tCH~v-. DcN+  
#s(^ J /3tCH~v-.DcN+ 
#s(^& /3tCH~v-.DcN+ 
#s(^" /3tCH~v-.DcN+ 
#s(^" /3QtCH~v-.QDcN+Q 
#s(^Ql /3tCH~v-.DcN+ 
#s(^4 /3tCH~v-.DcN+ 
#s(^4 /3tCH~v-.DcN+ 
#s(^( /3tCH~v-.DcN+ 
#s(^( /3:tCH~v-.:DcN+: 
#s(^:~ /3tCH~v-.DcN+ 
#s(^@ /3tCH~v-.DcN+ 
#s(^  /3tCH~v-.DcN+ 
#s(^  /3tCH~v-.DcN+ 
#s(^  /3tCH~v-.DcN+ 
#s(^, /3&tCH~v-.&DcN+& 
#s(^&V /3tCH~v-.DcN+ 
#s(^8 /30tCH~v-.0DcN+0 
#s(^0j /30tCH~v-.0DcN+0 
#s(^0fd
K"" d
N%^'' ^E8  Hs   M M$#M$