
    ci              
       X   d Z ddlZddlZddlZddlmZmZmZ ddlm	Z	 ddlm
Z ddlmZ ddlmZ dd	lmZ dd
lmZ ddlm
Z
  ej&                  d      Z G d de	j*                        ZdZefdede
j2                  fdZ G d dej6                        Z G d dej6                        Z G d dej6                        Zdee eef      deeef   fdZ!ejD                  eejF                  eejH                  eejJ                  ejL                  iZ'dedej6                  fdZ(dejR                  defdZ*dee
j2                     dee   fdZ+d e
jX                  d!e
jX                  d"e-d#e-ddf
d$Z.	 d)d%eee eef         d&ee   d'ee
j^                  j`                     de
jb                  fd(Z2y)*zDataset converters for evals.    N)AnyOptionalUnion)_common)types)ValidationError)override   )_evals_utils)_observability_data_converterz%vertexai_genai._evals_data_convertersc                   $    e Zd ZdZdZdZdZdZdZy)EvalDatasetSchemaz/Represents the schema of an evaluation dataset.geminiflattenopenaiobservabilityunknownN)	__name__
__module____qualname____doc__GEMINIFLATTENOPENAIOBSERVABILITYUNKNOWN     M/tmp/pip-target-z3e9_cxr/lib/python/vertexai/_genai/_evals_data_converters.pyr   r   #   s    9FGF#MGr   r   z*Error: Missing response for this candidatetextreturnc                     t        j                  t        j                  t        j                  |       g            S )z2Creates a ResponseCandidate with placeholder text.r    partsresponse)r   ResponseCandidategenai_typesContentPartr#   s    r   &_create_placeholder_response_candidater,   0   s3     ""$$K,<,<$,G+HI r   c                       e Zd ZdZdeeef   deej                  ej                  e
ej                  j                     ej                  f   fdZede
eeef      dej"                  fd       Zy)_GeminiEvalDataConverterz+Converter for dataset in the Gemini format.request_datar!   c           	         t        j                         }t        j                         }t        j                         }g }d|v r"t         j                  j	                  |d         }t        |j                  dg             D ]  \  }}t        |t              s't        d|dt        |      j                  d|      d|vrt        d|d|      |j                  t        j                  j                  t!        |      t         j                  j	                  |                    |r|j#                         }|j$                  r|j$                  j&                  nd	}	|	d
v r-|j$                  r|j$                  nt        j                         }nx|	dk(  rst        j                  |j$                        }|r=|j#                         }
|
j$                  r|
j$                  nt        j                         }nt        j                         }||||fS )z'Parses a request from a Gemini dataset.system_instructioncontentsz*Expected a dictionary for content at turn 
, but got : r%   z1Missing 'parts' key in content structure at turn turn_idcontentuser)r8   Nmodelr&   )r)   r*   r   r(   model_validate	enumerateget
isinstancedict	TypeErrortyper   
ValueErrorappendevalsMessagestrpopr7   role)selfr/   r1   prompt	referenceconversation_historyr6   content_dictlast_messagelast_message_rolesecond_to_last_messages              r   _parse_requestz'_GeminiEvalDataConverter._parse_request<   s    )002$$&++-	!</!,!4!4!C!C12" &/|/?/?
B/O%P 	!G\lD1\ 2 ; ;\K  l* .  !''##L'//>>|L $ 	"  /335L-9-A-A$$))v  !N2 $++ !(($,,. 
 #g-!33\=Q=QR	'-A-E-E-G* 299 /66(002  )002F)+?JJr   raw_datac           
         g }t        |      D ]  \  }}d|z  }|j                  di       }|j                  di       }| j                  |      \  }}	}
}g }t        |t              rO|j                  t        j                  t        j                  t        j                  |      g                   nt        |t              r	 t        j                  j                  |      }|j                  rh|j                  d   }|j                  rf|j                  t        j                  t        j                  j                  |j                                     n|j                  t!                      n|j                  t!                      t        j$                  |||||	|
      }|j                  |        t        j&                  |	      S # t"        $ r |j                  t!                      Y iw xY w)
6Converts a list of raw data into an EvaluationDataset.zgemini_eval_case_%srequestr'   r#   r$   r&   r   eval_case_idrI   	responsesrJ   r1   rK   
eval_cases)r;   r<   rP   r=   rE   rB   r   r(   r)   r*   r+   r>   GenerateContentResponser:   
candidatesr7   r,   	ExceptionEvalCaseEvaluationDataset)rH   rQ   rY   iitemrV   r/   response_datarI   r1   rK   rJ   rW   generate_content_response	candidate	eval_cases                   r   convertz _GeminiEvalDataConverter.convertv   s    
 * 5	)GAt014L88Ir2L HHZ4M ##L1"$ I--  ++!,!4!4#.#3#3#G"H" M40O#;;JJ) .
 1;;$=$H$H$K	$,,%,, % 7 7-8-@-@-O-O(1(9(9.&!" "(()O)QR   !G!IJ)###5%9I i(k5	)n &&*== ! O$$%K%MNOs   B,G"G65G6N)r   r   r   r   r>   rE   r   tupler)   r*   listr   rC   rD   r(   rP   r	   r^   re   r   r   r   r.   r.   9   s    58K4S> 8KeU[[  !	!? 8Kt ;>T#s(^ 4 ;>9P9P ;> ;>r   r.   c                   D    e Zd ZdZdeeeef      dej                  fdZ
y)_FlattenEvalDataConverterz4Converter for datasets in a structured table format.rQ   r!   c                 8   g }t        |      D ]4  \  }}t        |t              s't        d|dt	        |      j
                  d|      t        j                  |      }d|z  }|j                  dd      }|s|j                  dd      }|j                  dd      }|j                  d	d      }	|j                  d
d      }
|j                  dd      }|j                  dd      }|j                  dd      }|	st        d|z        |st        d|z        t        |t              r,t        j                  t        j                  |      g      }ngt        |t              r t        j                  j                  |      }n7t        |t        j                        r|}nt        d|dt	        |            d}t        |t              rg }t        |      D ]  \  }}t        |t        j                        r:|j!                  t"        j$                  j'                  t        |      |             Zt        |t              rZ	 t        j                  j                  |      }|j!                  t"        j$                  j'                  t        |      |             t*        j-                  d||t	        |              t        |	t              r5t#        j.                  t        j                  j                  |	            g}nt        |	t              rAt#        j.                  t        j                  t        j                  |	      g            g}nLt        |	t        j                        rt#        j.                  |	      g}nt        d|dt	        |	            d}|
rt        |
t              r4t#        j.                  t        j                  j                  |
            }nt        |
t              r@t#        j.                  t        j                  t        j                  |
      g            }n0t        |
t        j                        rt#        j.                  |
      }d}|rt        |t              r t        j                  j                  |      }nXt        |t              r,t        j                  t        j                  |      g      }nt        |t        j                        r|}d}|rLt        |t              r%i }|j1                         D ]  \  }}t        |t              r	 |D cg c]=  }t        |t              r)t"        j$                  j2                  j                  |      n|? }}t5        d |D              rt#        j6                  |      ||<   nt*        j-                  d||       t        |t"        j6                        r|||<   t        |t              r$	 t"        j6                  j                  |      ||<   t*        j-                  d||        nt*        j-                  d|       d}|rt        |t              rg }|D ]  }t        |t              r<	 t"        j$                  j:                  j                  |      }|j!                  |       Ot        |t"        j$                  j:                        r|j!                  |       t*        j-                  d |        nt*        j-                  d!|       t#        j<                  d$||||||||d"|}|j!                  |       7 t#        j>                  |#      S # t(        $ r#}t*        j-                  d|||       Y d}~d}~ww xY wc c}w # t8        $ r#}t*        j-                  d|||       Y d}~d}~ww xY w# t8        $ r#}t*        j-                  d|||       Y d}~d}~ww xY w# t8        $ r"}t*        j-                  d||       Y d}~d}~ww xY w)%rS   z(Expected a dictionary for item at index r3   r4   zeval_case_%srI   Nsourcehistoryr'   rJ   instructionrubric_groupsintermediate_eventsz(Response is required but missing for %s.z&Prompt is required but missing for %s.r#   r$   zInvalid prompt type for case r5   zqItem at index %s in 'history' column for case  %s is a dict but could not be validated as genai_types.Content: %szInvalid type in 'history' column for case %s at index %s. Expected genai_types.Content or dict, but got %s. Skipping this history item.r&   zInvalid response type for case c              3   d   K   | ](  }t        |t        j                  j                         * y wN)r=   r   rC   Rubric).0rs     r   	<genexpr>z4_FlattenEvalDataConverter.convert.<locals>.<genexpr>M  s)      '"() %/q%++2D2D$E'"s   .0)rubricsz;Invalid item type in rubric list for group '%s' in case %s.z8Failed to validate rubrics for group '%s' in case %s: %szAFailed to validate RubricGroup dict for group '%s' in case %s: %sz^Invalid type for rubric group '%s' in case %s. Expected list of rubrics, dict, or RubricGroup.z9Invalid type for rubric_groups in case %s. Expected dict.z:Failed to validate intermediate event dict for case %s: %szlInvalid type for intermediate_event in case %s. Expected list of dicts or list of types.evals.Event objects.z\Invalid type for intermediate_events in case %s. Expected list of types.evals.Event objects.)rV   rI   rW   rJ   rK   r1   rn   ro   rX   r   ) r;   r=   r>   r?   r@   r   copydeepcopyrF   rA   rE   r)   r*   r+   r:   rg   rB   r   rC   rD   r   loggerwarningr(   itemsrr   allRubricGroupr\   Eventr]   r^   ) rH   rQ   rY   r_   	item_dictr`   rV   prompt_dataconversation_history_datara   reference_datasystem_instruction_datarubric_groups_dataintermediate_events_datarI   rK   r6   r7   validated_contenterW   rJ   r1   rn   keyvaluert   validated_rubricsro   eventvalidated_eventrd   s                                    r   re   z!_FlattenEvalDataConverter.convert   sw   
%h/ o	)LAyi.$y/22I?  ==+D)A-L((8T2K"hhx6(,D(A% HHZ6M!XXk48N&*hh}d&C#!%/4!@'+xx0Et'L$  >M   <|K 
 +s+$,,K4D4D+4V3WXK.$,,;;KHK)<)<=$ =>[@QR  IM 3T:')$(12K(L $$GW!';+>+>?,33!KK//(+G(/ 0  $GT20;0C0C0R0R '1- 177 % 3 3,/L,= !4 !"  : )# M;$N -.++!,!4!4!C!CM!R	
 M3/++!,!4!4#.#3#3#G"H"	 M;+>+>?"44mLM	 ?@$}BUV  <@Ind3 % 7 7!,!4!4!C!CN!S!I  4 % 7 7!,!4!4#.#3#3#H"I"!I
  0C0CD % 7 7 PI@D&5t<)4)<)<)K)K/*&   7=)4)<)<*//5LMN*&   79L9LM)@&DHM!0$7$&M&8&>&>&@ 3
U%eT2" .35" )* ,6a+> ).(:(:(I(I!(L-.)/5" 1 5" $' '"->'" $" :?9J9J0A:&M#$6 %+NN(e(+()%& (u/@/@A16M#.'t4
"5:5F5F5U5U$)6"c 2 #NN!S # !	]3j NNS
 FJ'6=*,'!9 %eT2"27++2C2C2R2R$)3" !4 : :? K (u{{/@/@A/66u="NN!> !"	#0 NN>  )##%9#5+$7 I i(_o	)b &&*==g  / "NN!; !( , ! X5"* $- " &$^$'$%$%	!" !"" $- " &$g$'$%$%	!" !""> $- " &%3$%$%	!" !""su   A]7^;A^=A^4"^?4:_.	^%^^^	^<^77^<?	_+_&&_+.	`7``N)r   r   r   r   rg   r>   rE   r   r   r^   re   r   r   r   ri   ri      s.    >t>T#s(^ 4 t>9P9P t>r   ri   c            	           e Zd ZdZdeeeef      dee	e
j                     eej                  j                     e	e
j                     e	ej                     f   fdZedeeeef      dej$                  fd       Zy)_OpenAIDataConverterz9Converter for dataset in OpenAI's Chat Completion format.messagesr!   c                 V   d}d}d}g }|rX|d   j                  d      dv rBt        j                  t        j                  |d   j                  d            g      }|dd }t	        |      D ]  \  }}|j                  dd	      }|j                  dd
      }	|j                  t        j                  j                  t        |      t        j                  t        j                  |	      g|      |              |r|j                         }
|
j                  r&|
j                  j                  d	k(  r|
j                  }nc|
j                  rW|
j                  j                  dk(  r>t        j                  |
j                        }|r|j                         }|j                  }||||fS )zKParses a list of messages into instruction, history, prompt, and reference.Nr   rG   )system	developerr7   r#   r$   r
   r8    )r%   rG   )r6   r7   author	assistantr&   )r<   r)   r*   r+   r;   rB   r   rC   rD   rE   rF   r7   rG   r(   )rH   r   r1   rI   rJ   rK   r6   msgrG   r7   rM   rO   s               r   _parse_messagesz$_OpenAIDataConverter._parse_messages  s    "	!/3JJ!,!4!4"''Xa[__Y-GHI"  |H%h/ 	LGS7766*Dggi,G ''##L'//*//W=>T   $ 	  /335L##(<(<(A(AV(K%--%%,*>*>*C*C{*R!33\=Q=QR	'-A-E-E-G*3;;F!#7JJr   rQ   c                     g }t        |      D ]"  \  }}d|z  }d|vsd|vrt        j                  d|       +|j                  di       }|j                  di       }i }t	        |t
              rJ	 t        j                  |      }	t	        |	t              r|	}n t        j                  d|t        |	             nt	        |t              r|}|j                  dg       }
|j                  dg       }| j                  |
      \  }}}}||t        j                  d	|       g }|rt	        |t              rt	        |d
   t              r{|d
   j                  d      rg|d
   d   j                  dd      }|j                  t        j                  t!        j"                  t!        j$                  |      g                   n|j                  t'                      |j)                         D ci c]  \  }}|dvs|| }}}t        j*                  d||||||d|}|j                  |       % t        j,                  |      S # t        j                  $ r t        j                  d|       Y w xY wc c}}w )zHConverts a list of OpenAI ChatCompletion data into an EvaluationDataset.zopenai_eval_case_%srT   r'   z<Skipping case %s due to missing 'request' or 'response' key.z?Decoded response JSON is not a dictionary for case %s. Type: %szNCould not decode response JSON string for case %s. Treating as empty response.r   choiceszECould not determine a user prompt or reference for case %s. Skipping.r   messager7   r   r#   r$   r&   )rT   r'   rU   rX   r   )r;   ry   rz   r<   r=   rE   jsonloadsr>   r@   JSONDecodeErrorr   rg   rB   r   r(   r)   r*   r+   r,   r{   r]   r^   )rH   rQ   rY   r_   r`   rV   r/   response_data_rawra   loaded_jsonr   r   r1   rK   rI   rJ   rW   response_contentkvother_fieldsrd   s                         r   re   z_OpenAIDataConverter.convert  s    
 * S	)GAt014L$
$(>RTU 88Ir2L $R 8M+S1"&**->"?K!+t4(3, -	 -t4 1#''
B7H#''	26G $$X."$ ~)"3!
 Iw-wqz40AJNN9-#*1:i#8#<#<Y#K   ++!,!4!4#.#3#39I#J"K"   !G!IJ "&A:Q1Q1L   )###5%9 I i(gS	)j &&*==y ++ NN7Zs   0AI
/I:<I:
)I76I7N)r   r   r   r   rg   r>   rE   r   rf   r   r)   r*   r   rC   rD   r(   r   r	   r^   re   r   r   r   r   r     s    C)KT#s(^(< )K$$%U[[  !$$%(()	+B )KV X>T#s(^ 4 X>9P9P X> X>r   r   raw_datasetc                 ^   | st         j                  S | d   }t        |j                               }d|v r7|j	                  dd      }t        |t              r|dk(  rt         j                  S d|v rwd|v rs|j	                  di       }t        |t              rQd|v rM|j	                  d      }|r:t        |t              r*t        |d   t              rd|d   v rt         j                  S d|v r~d|v rz|j	                  di       }t        |t              rXd	|v rT|j	                  d	      }|rAt        |t              r1t        |d   t              rd
|d   v rd|d   v rt         j                  S ddhj                  |      sddhj                  |      rt         j                  S t         j                  S )z$Detects the schema of a raw dataset.r   formatr   r   rT   r'   r2   r%   r   rG   r7   rI   rJ   )r   r   setkeysr<   r=   rE   r   r>   rg   r   r   issubsetr   )r   
first_itemr   format_contentrequest_contentcontents_listmessages_lists          r   auto_detect_dataset_schemar   9  s     (((QJz !D4#"5nc*~/P$222DZ4/$..B7ot,1N+//
;M}d3}Q/6mA..,333DZ4/$..B7ot,1N+//
;M}d3}Q/6]1--)}Q?O2O,333*&&t,1 htn1 !((( (((r   dataset_schemac                 H    | t         v rt        |           S t        d| z        )z?Returns the appropriate dataset converter for the given schema.zUnsupported dataset schema: %s)_CONVERTER_REGISTRYrA   )r   s    r   get_dataset_converterr   q  s,     ,,">2449NJKKr   r7   c                     | rbt        | d      rVt        | j                  t              r<| j                  r0| j                  d   }t        |d      rt	        |j
                        S y)z6Safely extracts text from the first part of a content.r%   r   r    r   )hasattrr=   r%   rg   rE   r    )r7   
first_parts     r   _get_first_part_textr   {  sP     	GW%w}}d+MM]]1%
:v&z''r   rJ   c                 b    | r-t        | d      r!| j                  rt        | j                        S y)z,Safely extracts text from a reference field.r'   N)r   r'   r   )rJ   s    r   _get_text_from_referencer     s,     WY
3	8J8J#I$6$677r   	base_casecurrent_casecase_idxdataset_idxc           	         | j                   |j                   k7  rJt        | j                         dd }t        |j                         dd }t        j                  d|||||       t	        | j
                        }t	        |j
                        }t        | j
                        t        |j
                        k7  rt        j                  d||       y||k7  r2t        j                  d||t        |      dd t        |      dd        yy)z6Logs warnings if prompt or reference mismatches occur.N2   zPrompt mismatch for case index %d between base dataset (0) and dataset %d. Using prompt from base. Base prompt preview: '%s...', Dataset %d prompt preview: '%s...'zReference presence mismatch for case index %d between base dataset (0) and dataset %d. Using reference (or lack thereof) from base.zReference text mismatch for case index %d between base dataset (0) and dataset %d. Using reference from base.  Base ref: '%s...', Current ref: '%s...')rI   r   ry   rz   r   rJ   boolrE   )r   r   r   r   base_prompt_text_previewcurrent_prompt_text_previewbase_ref_textcurrent_ref_texts           r   _validate_case_consistencyr     s    <...#7	8H8H#I#2#N &:<;N;N&OPSQS&T#* $'
	
 -Y-@-@AM/0F0FGI D)?)?$@@# 	
 
*	* s# !#2&		
 
+r   raw_datasetsschemas
agent_infoc                    t        | t              st        dt        |       z        | rt	        d | D              st        d      |rt        |      t        |       k7  r$t        dt        |      dt        |       d      t        | d         }|dk(  r+t        j                  d       t        j                  g 	      S g }t        t        | |            D ]b  \  }\  }}t        |      |k7  r!t        d
|d|d|dt        |      d	      t        |      }|j                  |j                  |             d g }	|d   }
t!        |      D ]  }|
j"                  r|
j"                  |   nt        j$                         }g }|j&                  r|j                  |j&                  d          n3t        j                  d|       |j                  t)        d|z               |j+                  h dd      }t        |dd d      D ]  \  }}|j"                  r|j"                  |   nt        j$                         }t-        ||||       |j+                  h dd      }|j/                  |       |j&                  r|j                  |j&                  d          t        j                  d||       |j                  t)        d|d|              t        j$                  d|j0                  xs d|z  |j2                  ||j4                  |j6                  |j8                  ||j:                  d|}|	j                  |        t        j                  |		      S )a3  Merges multiple raw response datasets into a single EvaluationDataset.

    Assumes that each dataset in raw_datasets has responses corresponding
    to the same set of prompts, in the same order. The prompt, reference,
    system_instruction, and conversation_history are taken from the first dataset.
    z,Input 'raw_datasets' must be a list, got %s.c              3   <   K   | ]  }t        |t                y wrq   )r=   rg   )rs   dss     r   ru   z@merge_response_datasets_into_canonical_format.<locals>.<genexpr>  s     "OB:b$#7"Os   zAInput 'raw_datasets' cannot be empty and must be a list of lists.zBA list of schemas must be provided, one for each raw dataset. Got z schemas for z
 datasets.r   z@The first dataset has no evaluation cases. Result will be empty.rX   zQAll datasets must have the same number of evaluation cases. Base dataset (0) has z, but dataset z
 (schema: z) has .zLNo response found for base dataset (index 0) in case %s. Adding placeholder.z2Missing response from base dataset (0) for case %s>   rI   rJ   rW   rV   r1   ro   rK   T)excludeexclude_noner
   N)startz@No response found for dataset %s in case %s. Adding placeholder.zMissing response from dataset z
 for case zmerged_eval_case_%s)rV   rI   rW   rJ   r1   rK   r   ro   r   )r=   rg   r?   r@   r|   rA   lenry   rz   r   r^   r;   zipr   rB   re   rangerY   r]   rW   r,   
model_dumpr   updaterV   rI   rJ   r1   rK   ro   )r   r   r   num_expected_casesparsed_evaluation_datasetsr_   raw_ds_entryschema	convertermerged_eval_casesbase_parsed_datasetr   base_eval_casecandidate_responseseval_case_custom_columnsdataset_idx_offsetcurrent_parsed_dscurrent_ds_eval_casecurrent_ds_extra_attrsmerged_cases                       r   -merge_response_datasets_into_canonical_formatr     s    lD):T,=OO
 	
 s"O,"OOO
 	
 c'lc,&7714Ws<?PR
 	

 \!_-QN	
 &&"55@B%.s</I%J K!!L&| 22 &q&#l2CE 
 *&1	")))*;*;L*IJK /14Q7,- X. #--  **84! 	
 >@##&&~'?'?'BCNN&
  &&6H8S $2#<#<  $= $
  6?&qr*!6
 (	1 1
 %// ",,X6^^% ! ' 4h@R &:%D%D " &E &" %++,BC#--#**+?+I+I!+LMV&
 $**:-x9G(	T nn 
'44 0$x/!(()$..-@@!/!D!D! . B B
 '
 	  -qX.t "".?@@r   rq   )3r   rw   r   loggingtypingr   r   r   google.genair   r   r)   pydanticr   typing_extensionsr	   r   r   r   	getLoggerry   CaseInSensitiveEnumr   _PLACEHOLDER_RESPONSE_TEXTrE   r(   r,   EvalDataConverterr.   ri   r   rg   r>   r   r   r   r   r   ObservabilityDataConverterr   r   r*   r   r   r]   intr   rC   	AgentInfor^   r   r   r   r   <module>r      s%   $    ' '   - $ &  +  
		B	C33  J  +

y>|== y>xw> > > w>tG><99 G>T-)d38n%-)
c!"-)b 682##%B%]%]	 L%L##L+"5"5 # //0c]+
~~+
..+
 +
 	+

 
+
b 37HAtDcN+,HA#YHA ../HA 	HAr   