
    ci9                     ^   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	m
Z
mZmZmZmZmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dl mZ! d dl m"Z# d dl$m%Z% erd dl&Z'dZ(dZ) ejT                  e+      Z, G d dejZ                        Z. G d d      Z/d6dee0   de	egef   fdZ1	 d6dee2   de.fdZ3dee2de
e2ef   f   ddfdZ4de2ddfdZ5de2ddfdZ6de2ddfd Z7de2de2fd!Z8d"dd#e2d$ed%   ddfd&Z9	 	 	 	 d7d'e
e2e0f   d#e2d(ee2   d)ee2   d*ee2   d+eeee2e!jt                  f         ddfd,Z;d#e2d-e2ddfd.Z<	 	 	 	 	 d8d/ejz                  d0e2d1ee2   d(ee2   d)ee2   d*ee2   d+eeee2e!jt                  f         ddfd2Z>d3ee
e2e2f      d4dfd5Z?y)9    N)AnyCallableDictListLiteralOptionalTYPE_CHECKINGUnion)bigquery)storage)base)compat)initializer)utils)_ipython_utils)evaluation_service)_base)metric_prompt_template)blob_from_urizbq://zgs://c                   Z    e Zd ZdZej
                  Zej
                  ej                  ffZ	y)$_EvaluationServiceClientWithOverrideFN)
__name__
__module____qualname___is_temporaryr   V1_default_versiongapic_evaluation_servicesEvaluationServiceClient_version_map     @/tmp/pip-target-z3e9_cxr/lib/python/vertexai/evaluation/utils.pyr   r   5   s/    Myy II%==	
Lr"   r   c                   6    e Zd ZdZddee   fdZdefdZd Zy)	RateLimitera+  Helper class for rate-limiting requests to Vertex AI to improve QoS.

    Attributes:
        seconds_per_event: The time interval (in seconds) between events to
            maintain the desired rate.
        last: The timestamp of the last event.
        _lock: A lock to ensure thread safety.
    Nratec                     |r|dk  rt        d      d|z  | _        t        j                         | j                  z
  | _        t	        j
                         | _        y)a  Initializes the rate limiter.

        A simple rate limiter for controlling the frequency of API calls. This class
        implements a token bucket algorithm to limit the rate at which events
        can occur. It's designed for cases where the batch size (number of events
        per call) is always 1 for traffic shaping and rate limiting.

        Args:
            rate: The number of queries allowed per second.
        Raises:
            ValueError: If the rate is not positive.
        r   zRate must be a positive numberg      ?N)
ValueErrorseconds_per_eventtimelast	threadingLock_lock)selfr&   s     r#   __init__zRateLimiter.__init__J   sK     tqy=>>!$tIIK$"8"88	^^%
r"   returnc                     t        j                          }|| j                  z
  }|| j                  k\  r|| _        y| j                  |z
  S )zEChecks if an event can be admitted or calculates the remaining delay.r   )r*   r+   r)   )r/   nowtime_since_lasts      r#   _admitzRateLimiter._admit]   sC    iik		/d444DI))O;;r"   c                     | j                   5  | j                         }|dkD  r.t        j                  |       t        j                         | _        ddd       y# 1 sw Y   yxY w)z?Blocks the current thread until the next event can be admitted.r   N)r.   r5   r*   sleepr+   )r/   delays     r#   sleep_and_advancezRateLimiter.sleep_and_advanceg   sI    ZZ 	(KKMEqy

5! IIK			( 	( 	(s   AAA#N)	r   r   r   __doc__r   floatr0   r5   r9   r!   r"   r#   r%   r%   @   s(    &Xe_ &&< <(r"   r%   r&   r1   c                       fd}|S )z"Decorator version of rate limiter.c                 ^     t              t        j                          fd       }|S )Nc                  4    j                           | i |S r:   )r9   )argskwargslimitermethods     r#   wrapperz0rate_limit.<locals>._rate_limit.<locals>.wrapperv   s    %%'4*6**r"   )r%   	functoolswraps)rC   rD   rB   r&   s   ` @r#   _rate_limitzrate_limit.<locals>._rate_limits   s/    d#		 	+ 
!	+ r"   r!   )r&   rG   s   ` r#   
rate_limitrH   p   s     r"   api_base_path_overridec                 ~    t         j                  j                  t        t         j                  j                  |       S )zCreates a client for the evaluation service.

    Args:
      api_base_path_override: Optional. Override default api base path.

    Returns:
      Instantiated Vertex AI EvaluationServiceClient with optional
      overrides.
    )client_classlocation_overriderI   )r   global_configcreate_clientr   location)rI   s    r#    create_evaluation_service_clientrP      s7     $$229%33<<5 3  r"   sourcezpd.DataFramec                 0   	 ddl }t        | |j                        r| j	                         S t        | t
              r |j                  |       S t        | t              r| j                  t              rt        | t        t              d       S t        j                  j                  |       \  }}|j                         dd }|dk(  rt        |       S |dk(  rt!        |       S t#        d| d|  d	      t%        d
      # t        $ r t        d      w xY w)a  Loads dataset from various sources into a DataFrame.

    Args:
        source: The dataset source. Supports the following dataset formats:
        * pandas.DataFrame: Used directly for evaluation.
        * Dict: Converted to a pandas DataFrame before evaluation.
        * str: Interpreted as a file path or URI. Supported formats include:
            * Local JSONL or CSV files:  Loaded from the local filesystem.
            * GCS JSONL or CSV files: Loaded from Google Cloud Storage
                (e.g., 'gs://bucket/data.csv').
            * BigQuery table URI: Loaded from Google Cloud BigQuery
                (e.g., 'bq://project-id.dataset.table_name').

    Returns:
        The dataset in pandas DataFrame format.
    r   NgPandas is not installed. Please install the SDK using "pip install google-cloud-aiplatform[evaluation]"   jsonlcsvUnsupported file type:  from z]. Please provide a valid GCS path with `jsonl` or `csv` suffix or a valid BigQuery table URI.zUnsupported dataset type. Must be a `pd.DataFrame`, Python dictionary, valid GCS path with  `jsonl` or `csv` suffix or a valid BigQuery table URI.)pandasImportError
isinstance	DataFramecopydictstr
startswith
_BQ_PREFIX_load_bigquerylenospathsplitextlower_load_jsonl	_load_csvr(   	TypeError)rQ   pd_	extension	file_types        r#   load_datasetro      s   &
 &",,'{{}	FD	!r||F##	FC	 Z(!&Z):";<<ww''/9OO%ab)	v&&%V$$))F6( C' '  [
 	
7  
4
 	

s   D   Dfilepathc                 6   	 ddl }| j                  t              r2t	        |       } |j
                  t        j                  |      d      S t        | d      5 } |j
                  |d      cddd       S # t        $ r t        d      w xY w# 1 sw Y   yxY w)z.Loads data from a JSONL file into a DataFrame.r   NrS   T)linesr)	rY   rZ   r`   _GCS_PREFIX_read_gcs_file_contents	read_jsonioStringIOopen)rp   rk   file_contentsfs       r#   rh   rh      s    
 ;'/9r||BKK6dCC(C  	/A2<<.	/ 	/  
4
 	

	/ 	/s   A7 B7BBc                     	 ddl }| j                  t              r2t	        |       } |j
                  t        j                  |      d      S  |j
                  | d      S # t        $ r t        d      w xY w)z,Loads data from a CSV file into a DataFrame.r   NrS   utf-8)encoding)rY   rZ   r`   rt   ru   read_csvrw   rx   )rp   rk   rz   s      r#   ri   ri      sv    
 ;'/9r{{2;;}5HHr{{8g66  
4
 	

s   A! !A6table_idc                     t        j                  t        j                  j                        }|j                  |       }|j                  |      j                         S )z2Loads data from a BigQuery table into a DataFrame.)project)r   Clientr   rM   r   	get_table	list_rowsto_dataframe)r   bigquery_clienttables      r#   rb   rb      sH     ook.G.G.O.OPO%%h/E$$U+88::r"   c                 \   t        j                  t        j                  j                  t        j                  j
                        }| t        t              d j                  dd      \  }}|j                  |      }|j                  |      }|j                         j                  d      S )zReads the contents of a file from Google Cloud Storage.

    Args:
        filepath: The GCS file path (e.g., 'gs://bucket_name/file.csv')

    Returns:
        str: The contents of the file.
    r   credentialsN/rT   r}   )r   r   r   rM   r   r   rc   rt   split
get_bucketblobdownload_as_stringdecode)rp   storage_clientbucket_name	blob_pathbucketr   s         r#   ru   ru      s     ^^))11--99N &c+&6&89??QGK&&{3F;;y!D""$++G44r"   dfupload_gcs_pathrn   )rV   rU   c                 v   t        j                         5 }|dk(  r3t        j                  j	                  |d      }| j                  |       nL|dk(  r5t        j                  j	                  |d      }| j                  |dd       nt        d	| d
| d      t        ||       ddd       y# 1 sw Y   yxY w)zUploads the provided Pandas DataFrame to a GCS bucket.

    Args:
        df: The Pandas DataFrame to upload.
        upload_gcs_path: The GCS path to upload the data file.
        file_type: The file type of the data file.
    rV   zmetrics_table.csv)path_or_bufrU   zmetrics_table.jsonlrecordsT)r   orientrr   rW   rX   z?. Please provide a valid GCS path with `jsonl` or `csv` suffix.N)	tempfileTemporaryDirectoryrd   re   jointo_csvto_jsonr(   _upload_file_to_gcs)r   r   rn   temp_dirlocal_dataset_paths        r#   _upload_pandas_df_to_gcsr     s     
	$	$	& A(!#h8K!LII"4I5'!!#h8M!NJJ#5itJT))F?:K LQ Q 
 	O-?@A A As   BB//B8summary_metricscandidate_model_namebaseline_model_namedataset_urimetricsc                 4   d| i}|r||d<   |r||d<   |r||d<   |ri }|D ]|  }t        |t        j                        st        |j                  t        j
                        sC|j                  j                  |j                  j                  d||j                  <   ~ ||d<   t        j                         5 }	t        j                  j                  |	d      }
t        j                  |t!        |
d             t#        ||
       d	d	d	       y	# 1 sw Y   y	xY w)
z/Uploads the evaluation summary to a GCS bucket.r   r   r   r   )criteriarating_rubricmetric_descriptionszsummary_metrics.jsonwN)r[   metrics_base_ModelBasedMetric_raw_metric_prompt_templatemetric_prompt_template_base_MetricPromptTemplate	_criteria_rating_rubricmetric_namer   r   rd   re   r   jsondumpry   r   )r   r   r   r   r   r   summaryr   metricr   local_summary_paths              r#   !_upload_evaluation_summary_to_gcsr   !  s    	?G *>&')<%&!,  	F&,"@"@Aj22+AAG
 !' B B L L%+%G%G%V%V;#F$6$67	 *=%&		$	$	& A(WW\\(4JK		'4 2C89O-?@A A As   8ADDfilenamec                     t        j                  t        j                  j                  t        j                  j
                        }t        | |      j                  |       y )Nr   )uriclient)r   r   r   rM   r   r   r   upload_from_filename)r   r   r   s      r#   r   r   G  sH    ^^))11--99N on=RRr"   eval_resultdestination_uri_prefix	file_namec                    |st        j                          y| j                  y|j                  t              r|rt
        j                  j                  |      \  }}|j                         dd }	|dz   |z   }
|
dz   |z   }t        | j                  ||	       t        | j                  |
dz   ||||       t        j                  |j                  t              d          yyt        d| dt         d      )a  Uploads eval results to GCS destination.

    Args:
        eval_result: Eval results to upload.
        destination_uri_prefix: GCS folder to store the data.
        file_name: Optional. File name to store the metrics table.
        candidate_model_name: Optional. Candidate model name.
        baseline_model_name: Optional. Baseline model name.
        dataset_uri: Optional. URI pointing to the dataset.
        metrics: Optional. List of metrics used for evaluation.
    NrT   r   z/summary_metrics.jsonzUnsupported destination URI: z=. Please provide a valid GCS bucket URI prefix starting with .)r   (display_gen_ai_evaluation_results_buttonmetrics_tabler`   rt   rd   re   rf   rg   r   r   r   r   r(   )r   r   r   r   r   r   r   	base_namerm   rn   output_foldermetrics_table_paths               r#   upload_evaluation_resultsr   Q  s   ( "??A  (((5#%77#3#3I#> Iy!)!"-I2S89DM!.!4y!@$))+=y .++ 77$# CC"((5a8! ( +,B+C D}A
 	
r"   metric_column_mappingdatasetc                     i }|j                   D ]  }|||<   	 | rD| j                         D ]1  \  }}||v r#t        j                  d| d| d| d| d	       -|||<   3 |S )z7Initializes metric column mapping with dataset columns.zCannot override `z` column with `:z` mapping because `z` column is present in the evaluation dataset. `metric_column_mapping` cannot override keys that are already in evaluation dataset columns.)columnsitems_LOGGERwarning)r   r   !initialized_metric_column_mappingcolumnkeyvalues         r#    initialize_metric_column_mappingr     s     )+%// ;4:)&1;/557 		?JC77'uOC5% I!!$ &GG :?1#6		? -,r"   r:   )NNNN)NNNNN)@rE   rw   r   rd   r   r,   r*   typingr   r   r   r   r   r   r	   r
   google.cloudr   r   google.cloud.aiplatformr   r   r   r   google.cloud.aiplatform.utilsr   #google.cloud.aiplatform_v1.servicesr   r   vertexai.evaluationr   	eval_basevertexai.evaluation.metricsr   r   r   'google.cloud.aiplatform.utils.gcs_utilsr   rY   rk   ra   rt   Loggerr   r   ClientWithOverrider   r%   r<   rH   r_   rP   ro   rh   ri   rb   ru   r   _Metricr   r   
EvalResultr   r   r!   r"   r#   <module>r      s  $  	  	    U U U !   ( * / ) 8 3 = B 

$++h
5+C+C -( -(`Xe_ #0D " -1$SM)&3
#~tCH~563
3
l/# /. /"7 7 7 ;S ;^ ;5c 5c 5(AA),A9@9PA	A< +/)-!%@D#A#u*%#A#A #3-#A "#	#A
 ##A d5l&:&:!:;<=#A 
#AL     $*.)-!%@D2
%%2
2
 }2
 #3-	2

 "#2
 #2
 d5l&:&:!:;<=2
 
2
j-#DcN3->L-r"   