
    ciO-                        d Z ddlZddlZddlZddlmZmZmZ ddlm	Z	 ddl
mZ ddl
mZ ddlZddlmZ dd	lmZ dd
lmZ ddlmZ  ej,                  e      ZdZdZ G d d      Z G d d      Z G d dej:                        Zy)zUtility functions for evals.    N)AnyOptionalUnion)BaseApiClient)get_value_by_path)set_value_by_path   )_bigquery_utils)
_gcs_utils)_transformers)typeszgs://zbq://c            
       ~    e Zd ZdZdefdZdededeee	ee
f      e
f   fdZdeed	f   deee	ee
f      e
f   fd
Zy)EvalDatasetLoaderzBA loader for datasets from various sources, using a shared client.
api_clientc                     || _         t        j                  | j                         | _        t	        j
                  | j                         | _        y N)r   r   GcsUtils	gcs_utilsr
   BigQueryUtilsbigquery_utils)selfr   s     C/tmp/pip-target-z3e9_cxr/lib/python/vertexai/_genai/_evals_utils.py__init__zEvalDatasetLoader.__init__+   s6    $#,,T__=-;;DOOL    filepath	file_typereturnc                 ^   |j                  t              r.| j                  j                  ||      }|j	                  d      S |dk(  r)t        j                  |d      }|j	                  d      S |dk(  r)t        j                  |d      }|j	                  d      S t        d	| d
      )z3Loads data from a file into a list of dictionaries.recordsorientjsonlT)linescsvzutf-8)encodingzUnsupported file type: 'z#'. Please provide 'jsonl' or 'csv'.)	
startswith
GCS_PREFIXr   read_gcs_file_to_dataframeto_dictpd	read_jsonread_csv
ValueError)r   r   r   dfs       r   
_load_filezEvalDatasetLoader._load_file0   s     z*::8YOB::Y://G#\\($7zzz33e#[[G<zzz33 .yk :  r   sourcezpd.DataFramec                    t        |t        j                        r|j                  d      S t        |t              r|j                  t              r=| j                  j                  |t        t              d       }|j                  d      S t        j                  j                  |      \  }}|j                         dd }|dk(  r| j                  |d      S |dk(  r| j                  |d      S t        d| d| d	      t        d
      )z?Loads dataset from various sources into a list of dictionaries.r   r    Nr	   r"   r$   zUnsupported file type: z from zq. Please provide a valid GCS path with `jsonl` or `csv` suffix, a local file path, or a valid BigQuery table URI.zUnsupported dataset type. Must be a `pd.DataFrame`, Python a valid GCS path with `jsonl` or `csv` suffix, a local file path, or a valid BigQuery table URI.)
isinstancer*   	DataFramer)   strr&   	BQ_PREFIXr   load_bigquery_to_dataframelenospathsplitextlowerr/   	TypeError)r   r0   r.   _	extensionr   s         r   loadzEvalDatasetLoader.loadD   s    fbll+>>>33$  +((CC3y>+, zzz3377++F3LAy!)!"-IG#vw77e#vu55-i[vh GH H  = r   N)__name__
__module____qualname____doc__r   r   r4   r   listdictr   r/   r?    r   r   r   r   (   s}    LM= M
(+	tDcN#S(	)(C/0	tDcN#S(	)r   r   c            
       x   e Zd ZdZe	 ddeeeef   e	f   de
eeef      deeef   fd       Ze	 ddeeeef   e	f   de
eeef      deeef   fd       Ze	 ddeeeef   e	f   de
eeef      deeef   fd       Ze	 ddeeeef   e	f   de
eeef      deeef   fd	       Ze	 ddeeeef   e	f   de
eeef      deeef   fd
       Ze	 ddeeeef   e	f   de
eeef      deeef   fd       Ze	 ddeeeef   e	f   de
eeef      deeef   fd       Zedeeef   ded   deeef   fd       Zy)BatchEvaluateRequestPreparerzPrepares data for requests.Nfrom_objectparent_objectr   c                     i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )N
gcs_sourcebigquery_sourcegetvsetvrI   rJ   	to_objects      r   _EvaluationDataset_to_vertexz9BatchEvaluateRequestPreparer._EvaluationDataset_to_vertexi   sn    
 %'	l^,8[<.1 /01="#[#4"56 r   c                    i }t        | dg      t        |ddgt        | dg             t        | dg      t        |ddgt        | dg             t        | dg      t        |ddgt        | dg             t        | dg      t        |dd	gt        | dg             t        | d
g      t        |g dt        | d
g             |S )Nprompt_templatepointwise_metric_specjudge_modelautorater_configautorater_modeljudge_model_sampling_countsampling_countjudge_model_system_instructionsystem_instructionreturn_raw_output)rV   custom_output_format_configr^   rN   rQ   s      r   _Metric_to_vertexz.BatchEvaluateRequestPreparer._Metric_to_vertex   s   
 %'	/01=(*;<[#4"56 m_-9#%67[=/2 :;<H#%56[#?"@A >?@L(*>?[#C"DE 123?
 [#6"78 r   c                 V    i }t        | dg      t        |dgt        | dg             |S )Ngcs_destinationgcsDestinationrN   rQ   s      r   _OutputConfig_to_vertexz4BatchEvaluateRequestPreparer._OutputConfig_to_vertex   sA    
 %'	/01=!"[#4"56 r   c           	          i }t        | ddg      t        |dgt        | ddg             t        | ddg      t        |dgt        | ddg             |S )NdatasetrL   rM   rN   rQ   s      r   _EvaluationDataset_from_vertexz;BatchEvaluateRequestPreparer._EvaluationDataset_from_vertex   sx    
 %'	i67C[9l";< i):;<H"#[9.?"@A r   c                     i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             |S )Nr[   samplingCountflip_enabledflipEnabledrY   autoraterModelrN   rQ   s      r   _AutoraterConfig_to_vertexz7BatchEvaluateRequestPreparer._AutoraterConfig_to_vertex   s    
 %'	./0<_-tKBRAS/TUn-.:]OT+?O-PQ/01=!"[#4"56 r   c           
         i }t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      t        |dgt        | dg             t        | dg      -t        |dgt        j                  t        | dg      |             |S )Nnamemetadatadoneerrorresponse)rO   rP   rH   rg   rQ   s      r   $EvaluateDatasetOperation_from_vertexzABatchEvaluateRequestPreparer.EvaluateDatasetOperation_from_vertex   s    
 %'	fX&2VHd;&ABj\*6ZL${ZL*IJfX&2VHd;&ABgY'3WItK''CDj\*6,KKzl3Y r   c                 L   i }t        | dg      -t        |dgt        j                  t        | dg      |             t        | dg      ;t        |dgt        | dg      D cg c]  }t        j	                  ||       c}       t        | dg      -t        |dgt        j                  t        | dg      |             t        | dg      -t        |dgt        j                  t        | dg      |             t        | dg      t        |dgt        | dg             |S c c}w )Nrf   metricsoutput_configoutputConfigrX   autoraterConfigconfig)rO   rP   rH   rS   r`   rd   rm   )rI   rJ   rR   items       r   *EvaluateDatasetRequestParameters_to_vertexzGBatchEvaluateRequestPreparer.EvaluateDatasetRequestParameters_to_vertex  sH   
 %'	i[)5,IIyk2I i[)5 !%[9+ > 1BB4S o./; ,DD&78) 012>"#,GG'9&:;Y hZ(4XJ[8*(EF7s   !D!request_dictresolved_metricsztypes.MetricSubclassc                 :    t        j                  |d      | d<   | S )a8  Prepares the metric payload for the evaluation request.

        Args:
            request_dict: The dictionary containing the request details.
            resolved_metrics: A list of resolved metric objects.

        Returns:
            The updated request dictionary with the prepared metric payload.
        T)set_default_aggregation_metricsrv   )r   	t_metrics)r}   r~   s     r   prepare_metric_payloadz3BatchEvaluateRequestPreparer.prepare_metric_payload8  s&     #0"9"9d#
Y r   r   )r@   rA   rB   rC   staticmethodr   rE   r4   r   objectr   rS   r`   rd   rg   rm   rt   r|   rD   r   rF   r   r   rH   rH   f   s   % 374S>612S#X/ 
c3h ,  37-4S>612-S#X/- 
c3h- -^  374S>612S#X/ 
c3h   374S>612S#X/ 
c3h ,  374S>612S#X/ 
c3h (  374S>612S#X/ 
c3h 8  37-4S>612-S#X/- 
c3h- -^ 38n8<=S8T	c3h r   rH   c                   >    e Zd ZdZej
                  deddfd       Zy)EvalDataConverterz+Abstract base class for dataset converters.raw_datar   ztypes.EvaluationDatasetc                     t               )z8Converts a loaded raw dataset into an EvaluationDataset.)NotImplementedError)r   r   s     r   convertzEvalDataConverter.convertN  s     "##r   N)r@   rA   rB   rC   abcabstractmethodr   r   rF   r   r   r   r   K  s,    5$ $(A $ $r   r   )rC   r   loggingr8   typingr   r   r   google.genai._api_clientr   google.genai._commonr   rO   r   rP   pandasr*    r
   r   r   r   	getLoggerr@   loggerr'   r5   r   rH   ABCr   rF   r   r   <module>r      sy    # 
  	 ' ' 2 : :      
		8	$ 
	; ;|b bJ$ $r   