iLdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z e jjdddlmZddlZddlZddlZddlZddlZddlZddlmZmZddlmZmZddlmZmZmZdd l m Z m!Z!dd l"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3ejhejjd ejldZ7e2dZ8e9Z:e9Z;e9Z<Gdde#Z=Gdde#Z>Gdde#Z?Gdde#Z@eGddZAeGddZBeGddZCGdd ZDGd!d"ZEGd#d$ZFGd%d&ZGGd'd(ZHGd)d*ZId+ZJeKd,k(rejeJyy)-a rlm_01_preference_learning.py - Preference Learning System for AIVA Queen RLHF This module implements a comprehensive preference learning system for AIVA Queen's Reinforcement Learning from Human Feedback (RLHF) pipeline. It enables the collection, storage, modeling, and active querying of human preferences. Key Components: - PreferenceCollector: Gather and validate human preference annotations - PreferenceDataset: Persistent storage and retrieval of preference pairs - BradleyTerryModel: Statistical model for preference prediction - PreferenceLearner: Online and batch learning from preferences - ActivePreferenceQuery: Information-theoretic selection of comparisons - PreferenceIntegrator: Bridge preferences into the training pipeline Reference Papers: - Bradley-Terry Model (1952): Rank analysis of incomplete block designs - Christiano et al. (2017): Deep RL from Human Preferences - Rafailov et al. (2023): DPO - Direct Preference Optimization Author: Genesis-OS AIVA Queen RLM Module Version: 1.0.0 ) annotationsNz)/mnt/e/genesis-system/data/genesis-memory)PostgresConfig)ABCabstractmethod) defaultdictdeque) dataclassfieldasdict)datetime timedelta)Enumauto)Path) AnyCallableDictGenericIteratorListOptionalProtocolSetTupleTypeVarUnionz4%(asctime)s - %(name)s - %(levelname)s - %(message)s)levelformatzAIVA.PreferenceLearningTc,eZdZdZdZdZdZdZdZdZ dZ y ) PreferenceStrengthz+Strength of preference between two options.rN) __name__ __module__ __qualname____doc__STRONGLY_PREFER_APREFER_ASLIGHTLY_PREFER_ATIESLIGHTLY_PREFER_BPREFER_BSTRONGLY_PREFER_BJ/mnt/e/genesis-system/AIVA/queen_outputs/rlm/rlm_01_preference_learning.pyr!r!Ss,5H CHr4r!cVeZdZdZeZeZeZeZeZ y)FeedbackQualityzQuality assessment of feedback.N) r(r)r*r+rHIGHMEDIUMLOWSPAMATTENTION_CHECK_FAILEDr3r4r5r7r7^s)) 6D VF &C 6D!Vr4r7cHeZdZdZeZeZeZeZy)LearningStrategyz)Strategy for preference learning updates.N) r(r)r*r+rONLINE MINI_BATCH FULL_BATCH STREAMINGr3r4r5r>r>gs!3 VFJJIr4r>cVeZdZdZeZeZeZeZeZ y) QueryStrategyz(Strategy for active preference querying.N) r(r)r*r+r UNCERTAINTYINFORMATION_GAIN DIVERSITY DISAGREEMENTHYBRIDr3r4r5rDrDos)2&KvI6L VFr4rDceZdZUdZded<ded<ded<dZded <eej Zd ed <dZ d ed<ee Z ded<ddZ ddZ ddZeddZy)Responsez A model response to be compared. ResponseIdidstrprompttextN Optional[str]model_iddefault_factoryfloat timestampzOptional[List[float]] embeddingDict[str, Any]metadatac,t|jSN)hashrMselfs r5__hash__zResponse.__hash__sDGG}r4cVt|tsy|j|jk(S)NF) isinstancerKrM)r^others r5__eq__zResponse.__eq__s"%*ww%((""r4c|j|j|j|j|j|j dS)NrMrOrPrRrVrYrer]s r5to_dictzResponse.to_dicts6''kkII    r4c ||d|d|d|jd|jdtj|jdiS)NrMrOrPrRrVrYre)gettimeclsdatas r5 from_dictzResponse.from_dictsRDz>fXXj)hh{DIIK8XXj"-   r4returnint)rbobjectroboolrorX)rlrXroz 'Response')r(r)r*r+__annotations__rRr rirVrWdictrYr_rcrf classmethodrmr3r4r5rKrK|sn*N K I"Hm"TYY7Iu7'+I$+$T:Hn:#    r4rKcHeZdZUdZded<ded<ded<ded<d ed <d ed <eej Zded<dZded<dZ ded<e jZ ded<ee Zded<dZded<ed"dZed"dZed#dZed#dZd$d Zed%d!Zy)&PreferencePairz5A pair of responses with human preference annotation. ComparisonIdrMrNrOrK response_a response_br! preference AnnotatorId annotator_idrSrUrV? confidenceNrQ reasoningr7qualityrXrYFrris_attention_checkc|jjdkDr |jS|jjdkr |jSy)z+Get the winning response, or None for ties.rN)r|valuerzr{r]s r5winnerzPreferencePair.winner? ?? 1 $?? " __ " "Q &?? "r4c|jjdkDr |jS|jjdkr |jSy)z*Get the losing response, or None for ties.rN)r|rr{rzr]s r5loserzPreferencePair.loserrr4c4|jjdz S)z,Get normalized preference margin in [-1, 1].@)r|rr]s r5marginzPreferencePair.margins$$s**r4c&|jdzdz S)z;Get probability label for A being preferred (for training).r@)rr]s r5labelzPreferencePair.labels c!S((r4c p|j|j|jj|jj|j j |j|j|j|j|jj |j|jd S)N rMrOrzr{r|r~rVrrrrYr)rMrOrzrfr{r|namer~rVrrrrYrr]s r5rfzPreferencePair.to_dicts''kk//113//113//.. --//||(( "&"9"9  r4c||d|dtj|dtj|dt|d|d|jdt j|jdd |jd t |jd d |jd i|jdd S)NrMrOrzr{r|r~rVrrrrr9rYrFr)rKrmr!rhrir7rjs r5rmzPreferencePair.from_dictsDz>))$|*<=))$|*<=)$|*<=n-hh{DIIK8xx c2hh{+#DHHY$ABXXj"-#xx(@average_time_seconds quality_scorerS last_activeFrr is_trustedzDict[str, int]preferences_by_typeNch|xjdz c_tj|_d}d|z |jz||zz|_d|z |jz||zz|_|2|j}|dz |j zt |z|z |_d|jzd|j zzdtd|jzzdd|jcxkrdkrnndnd zz|_ |jd k\xr |jd k\xr|j d k\|_ y) z%Update profile from a new annotation.r$皙?N333333?g?ri,r ffffff?g?) rrirrrrrUrminrr)r^passed_attentionr time_spentalphans r5update_from_annotationz'AnnotatorProfile.update_from_annotationsO !#99;#$u90G0G"G%R\J\"\%&Y$2K2K$KeV`N`$`!  '&&AQ$8885AQ;RR.D * $%% % $00 0 1 #c4223 3 4 !d77=#=33 G H   " "b ( 2   # % 2  * *c 1 r4)Nrr)rzOptional[bool]rrUrrUroNone)r(r)r*r+rtrrrrrrr rirrrurrr3r4r5rrs<OsNE'*u* ##"&%&M5tyy9K9J*/*EE,0 " (" "  "  " r4rceZdZdZ d d dZddZddZ d ddZ d ddZ ddZ e jd f dd Z dd Z y)PreferenceCollectora" Collects and validates human preference annotations. Features: - Multi-source preference collection (direct, comparison, implicit) - Quality control with attention checks - Annotator reliability tracking - Spam and noise filtering - Async batch collection c||_||_||_||_||_i|_i|_|j|_g|_ tj|_ tjdy)a Initialize preference collector. Args: attention_check_frequency: Frequency of attention check insertions min_time_seconds: Minimum acceptable annotation time max_time_seconds: Maximum acceptable annotation time spam_threshold: Threshold for flagging spam annotators require_reasoning: Whether to require reasoning for preferences zPreferenceCollector initializedN)attention_check_frequencymin_time_secondsmax_time_secondsspam_thresholdrequire_reasoning_annotator_profiles_pending_comparisons_generate_attention_checks_attention_checks_collected_pairs threadingLock_lockloggerinfo)r^rrrrrs r5__init__zPreferenceCollector.__init__6sq$*C& 0 0,!2HJ HJ!7;7V7V7X68^^%  56r4c~dddtjddddtjddd d tjdgS) z@Generate attention check comparisons with known correct answers.zWhat is 2 + 2?zThe answer is 4.zThe answer is banana.)rOrzr{correctzWrite a greeting.zHello! Nice to meet you.zasdfghjkl qwertyzExplain what water is.zI don't know anything.zNWater is H2O, a molecule consisting of two hydrogen atoms and one oxygen atom.)r!r,r2r]s r5rz.PreferenceCollector._generate_attention_checksVsW+05-??  .80-??  36n-??    r4cn||jvrt||j|<|j|S)z Get or create annotator profile.)rM)rr)r^r~s r5get_annotator_profilez)PreferenceCollector.get_annotator_profilems6 t77 75E5VD $ $\ 2'' 55r4Nc 8ttj}tj|jk}|rtj |j }tttj|d|d}tttj|d|d}|d}|j5||||xsitj||rdndgd|j|<ddd|S#1swY|SxYw)a Create a new comparison task. Args: prompt: The prompt response_a: First response response_b: Second response metadata: Additional metadata Returns: Comparison ID rOrzrMrOrPr{rN)rOrzr{rY created_atrcorrect_answerr) rNuuiduuid4randomrchoicerrKrrir)r^rOrzr{rY comparison_idrchecks r5create_comparisonz%PreferenceCollector.create_comparisonss&DJJL) $]]_t/M/MM MM$"8"89E!tzz|$X<(J "tzz|$X<(J 8_F ZZ  (($N"iik&86H% "2d! 8D % %m 4  s 6DDcK|j5||jvr"tjd| dddy|j|}|dDcgc]}|d } }|| vr"tjd| dddy ddd|j |||||} d} drr|d} |j d kDxr| j d kDxs |j d kxr| j d k} | s)t j} tjd |d |j|} | j| || |||||| tjd }|j5|dj|ddd| t jt jfvrtttt!j"|d|d|d||||| |d|d }|j5|j$j|dddd|fSycc}w#1swYxYw#1swYxYw#1swYd|fSxYww)a Submit a preference annotation. Args: comparison_id: ID of the comparison annotator_id: ID of the annotator preference: The preference choice confidence: Annotator's confidence reasoning: Optional reasoning time_spent_seconds: Time spent on annotation Returns: Tuple of (accepted, resulting_pair) zUnknown comparison: N)FNrr~zDuplicate annotation from rrrz Annotator z failed attention check)rrr)r~r|rrrrrVrOrzr{rY) rMrOrzr{r|r~rrrrYrT)rrrwarning_assess_qualityrr7r<rrriappendr8r9rxrNrrr)r^rr~r|rrtime_spent_seconds comparisonaexisting_annotatorsrrrprofile annotationpairs r5submit_preferencez%PreferenceCollector.submit_preferences.ZZ #D$=$==!5m_EF" # # 22=AJ?I>W"X1^#4"X "X22!;L>JK" # #3 #&& *j)=O    * + !12G!!A%;'--!*;=!!A%;'--!*; $)@@L>9PQR,,\:&&-!) ' )$$", ZZ 9 } % , ,Z 8 9 ++_-C-CD D!tzz|$!(+%l3%l3%)%##J/#-.B#C D 3%%,,T2 3: E#Y # #d 9 9& 3: sq I(H' IH' H""H' I C#I.H4A6I9I I"H''H1,I4H=9II  Ic|j|}d}||jkr|dz }||jkDr|dz }|dkr|tjk(r|dz }|j r|s|dz }|j |jkr|dz }|dk\rtjS|dk\rtjS|dk\rtjStjS)z.Assess the quality of a preference annotation.rr#r$rr") rrrr!r/rrrr7r;r:r9r8)r^r~r|rrrr spam_signalss r5rz#PreferenceCollector._assess_quality s,,\:  -- - A L -- - A L   .@.D.D D A L  ! !) A L  4#6#6 6 A L 1 "'' ' Q "&& & Q ")) )"'' 'r4Tc tjdtjdtjdtjdtj di}||}|j 5|jDcgc]}||j|k\r|}}|r|Dcgc]}|jr|}}|cdddScc}wcc}w#1swYyxYw)z3Get collected preference pairs filtered by quality.r"r#r$rN) r7r8r9r:r;r<rrrr)r^ min_qualityexclude_attention_checks quality_rankmin_rankppairss r5get_collected_pairsz'PreferenceCollector.get_collected_pairs5s  !  " "A     !  2 2A   , ZZ 00 *h6E ($)Fq1E1EFF   G  s0$C3B9CB>(B>,C9 CC c 8|j5|j}t|jj }dddt t t Dcic]!jtfd|D#c}tDcic]!jtfd|D#c}|r-tj|Dcgc]}|jc}ndtd|DdS#1swYxYwcc}wcc}wcc}w)zGet collection statistics.Nc3BK|]}|jk(sdywr$N)r).0rqs r5 z5PreferenceCollector.get_statistics..Ys?! QA?c3BK|]}|jk(sdywr)r|)rrss r5rz5PreferenceCollector.get_statistics..]sB! 0AABrrc3:K|]}|jsdywr)r)rrs r5rz5PreferenceCollector.get_statistics..as%JAQ\\a%Js) total_pairstotal_annotatorsquality_distributionpreference_distributionrtrusted_annotators) rrlistrvalueslenr7rsumr! statisticsmeanr)r^rprofilesrrrs `` r5get_statisticsz"PreferenceCollector.get_statisticsOs ZZ ?))ED44;;=>H ? u: #H )%?u???% ,(BuBBB(UZ*//2OA1<<2O"P_`"%%J%J"J   ? ?%(3Ps0D$&D &D DD )rrg@rF) rrUrrUrrUrrUrrr)roList[Dict[str, Any]])r~r}rorr[) rOrNrzrKr{rKrYzOptional[Dict[str, Any]]rory)rNr)rryr~r}r|r!rrUrrQrrUroz%Tuple[bool, Optional[PreferencePair]]) r~r}r|r!rrUrrQrrUror7)rr7rrrroList[PreferencePair]rs)r(r)r*r+rrrrrrr7r9rrr3r4r5rr*sd ,/"%"' #"' 7#(7 7 7  7  7@ .6.2 111 1 + 1  1p #'$(a#a"a' a  a ! a"a /aF*(!*('*( *( ! *(  *( *(\(7'='=)-$#'  4 r4rceZdZdZ d ddZdZddZddZddZddZ dd Z dd Z dd Z dd Z d dd ZddZddZddZed d!dZy)"PreferenceDataseta Persistent storage and retrieval of preference pairs. Features: - PostgreSQL-backed persistence (Elestio) - Efficient batch operations - Filtering and sampling - Cross-annotator agreement computation - Export/import capabilities Nc:||_i|_tt|_tt|_t j|_|j|jtjdt|jdy)z Initialize preference dataset. Args: db_path: Deprecated (ignored). Uses Elestio PostgreSQL. min_confidence: Minimum confidence threshold for retrieval z#PreferenceDataset initialized with z pairsN)min_confidence_pairsrr _by_prompt _by_responserrr_init_database_load_from_databaserrr)r^db_pathrs r5rzPreferenceDataset.__init__usr-:< 9DT9JBMdBS^^%     " 9#dkk:J9K6RSr4cPtjditjS)z)Get a PostgreSQL connection from Elestio.r3)psycopg2connectrget_connection_paramsr]s r5 _get_connzPreferenceDataset._get_connsI."F"F"HIIr4c|j} |j5}|jdddd|j|j y#1swY*xYw#|j wxYw)zCEnsure PostgreSQL tables exist (tables should already exist in PG).a  CREATE TABLE IF NOT EXISTS pl_preference_pairs ( id TEXT PRIMARY KEY, prompt TEXT NOT NULL, response_a_id TEXT NOT NULL, response_a_text TEXT NOT NULL, response_b_id TEXT NOT NULL, response_b_text TEXT NOT NULL, preference TEXT NOT NULL, annotator_id TEXT NOT NULL, timestamp DOUBLE PRECISION, confidence REAL DEFAULT 1.0, reasoning TEXT, quality TEXT, metadata TEXT, is_attention_check INTEGER DEFAULT 0 ) N)rcursorexecutecommitclose)r^conncurs r5rz PreferenceDataset._init_databasesd~~  #  & KKM JJL+  * JJLs!A)AA)A&"A))A;c|j} |jtjj5}|j d|D]k}t |d|dt|d|d|dt|d|d|d t|d |d |d |d |d|dr t|dntj|drtj|dnit|d }||j|j<|j |j"j%|j|j&|j(jj%|j|j&|j*jj%|jn ddd|j-y#1swYxYw#|j-wxYw)z"Load existing pairs from database.)cursor_factoryz!SELECT * FROM pl_preference_pairsrMrO response_a_idresponse_a_textr response_b_idresponse_b_textr|r~rVrrrrYrrN)rrr extrasRealDictCursorrrxrKr!r7r9jsonloadsrrrrMrrOrrrzr{r)r^rrrowrs r5rz%PreferenceDataset._load_from_databases~~ HOO,J,JK Js ?@JC)t9"8}#+"?3#&x=!$%6!7$ $,"?3#&x=!$%6!7$ $6c,6G#H%(%8"%k"2#&|#4"%k"2CFy>I ?WfWmWm@CJC O! JJL? J J> JJLs#*G%FGG%G"G%%G7c|j5||j|j<|j|jj |j|j |jjj |j|j |jjj |jddd|j||jS#1swY&xYw)z%Add a preference pair to the dataset.N) rrrMrrOrrrzr{ _save_pairr^rs r5addzPreferenceDataset.adds ZZ B#'DKK OODKK ( / / 8   doo00 1 8 8 A   doo00 1 8 8 A  B ww B Bs CC66C?cXg}|D]"}|j|j|$|S)zAdd multiple preference pairs.)rr$)r^ridsrs r5 add_batchzPreferenceDataset.add_batchs/ 'D JJtxx~ & ' r4c|j} |j5}|jd|j|j|j j|j j |jj|jj |jj|j|j|j|j|jjtj |j"t%|j&fddd|j)|j+y#1swY*xYw#|j+wxYw)zSave pair to database.a INSERT INTO pl_preference_pairs (id, prompt, response_a_id, response_a_text, response_b_id, response_b_text, preference, annotator_id, timestamp, confidence, reasoning, quality, metadata, is_attention_check) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (id) DO UPDATE SET prompt = EXCLUDED.prompt, response_a_id = EXCLUDED.response_a_id, response_a_text = EXCLUDED.response_a_text, response_b_id = EXCLUDED.response_b_id, response_b_text = EXCLUDED.response_b_text, preference = EXCLUDED.preference, annotator_id = EXCLUDED.annotator_id, timestamp = EXCLUDED.timestamp, confidence = EXCLUDED.confidence, reasoning = EXCLUDED.reasoning, quality = EXCLUDED.quality, metadata = EXCLUDED.metadata, is_attention_check = EXCLUDED.is_attention_check N)rrrrMrOrzrPr{r|rr~rVrrrrdumpsrYrprrr)r^rrrs r5r"zPreferenceDataset._save_pairs~~( $ # *GGKKOO&&OO((OO&&OO((OO((%%NNOONNLL%%JJt}}-//0)#$ J KKM JJLO$ $ N JJLs#EDE'EEEE.c8|jj|S)zGet a preference pair by ID.)rrh)r^pair_ids r5rhzPreferenceDataset.gets{{w''r4c|j5|jj|g}|Dcgc] }||jvs|j|"c}cdddScc}w#1swYyxYw)z&Get all preference pairs for a prompt.N)rrrhr)r^rOpair_idspids r5 get_by_promptzPreferenceDataset.get_by_promptsc ZZ P**626H08OC4;;G6 GG>GG c,t|jSr[)rrr]s r5__len__zPreferenceDataset.__len__Ms4;;r4cHt|jjSr[)iterrrr]s r5__iter__zPreferenceDataset.__iter__PsDKK&&())r4c|j5|jjDcgc]}|j}}dddt |d5}t j tjjd|ddddtjdtd|ycc}w#1swYxYw#1swY?xYw)zExport dataset to JSON file.Nw)r exported_atr#)indentz Exported z pairs to ) rrrrfopenrdumpr now isoformatrrr)r^filepathrrfs r5 export_jsonzPreferenceDataset.export_jsonSs ZZ @*.++*<*<*>?QQYY[?E? @(C  `A IIhlln6N6N6PQST]^ _ `  iE |:hZ@A @ @ @ ` `s(CCC B) = sigma(r_A - r_B) Where r_A and r_B are learned "strength" parameters for each response, and sigma is the sigmoid function. Features: - Online and batch parameter estimation - Regularization (L2 penalty) - Uncertainty estimation via Laplace approximation - Efficient incremental updates c||_||_|_||_t fd|_t t |_d|_t t|_ y)a@ Initialize Bradley-Terry model. Args: learning_rate: Learning rate for parameter updates regularization: L2 regularization strength prior_strength: Prior strength for all responses use_confidence_weighting: Weight updates by annotator confidence cSr[r3prior_strengthsr5z,BradleyTerryModel.__init__..s~r4rN) learning_rateregularizationr^use_confidence_weightingr _strengthsrp_counts _pair_countrU _hessian_diag)r^r`rar^rbs ` r5rzBradleyTerryModel.__init__sV +,,(@%3>?U3V.9#.> 7B%6Hr4c|dk\rtj| }dd|zz Stj|}|d|zz S)zNumerically stable sigmoid.rr$)mathexp)r^xzs r5_sigmoidzBradleyTerryModel._sigmoidsA 6! AA;  AA; r4cf|j|}|j|}|j||z S)z Predict probability that A is preferred over B. Args: response_a: ID of response A response_b: ID of response B Returns: P(A > B) in [0, 1] )rcrlr^rzr{ strength_a strength_bs r5predict_preferencez$BradleyTerryModel.predict_preferences4__Z0 __Z0 }}Z*455r4cdt|j||jzdz }dt|j||jzdz }||z}|j||}| t j |dzzd|z t j d|z dzzz }|t j |zS)a+ Get uncertainty in preference prediction. Uses Laplace approximation to estimate prediction variance. Args: response_a: ID of response A response_b: ID of response B Returns: Uncertainty score (higher = more uncertain) rgư>绽|=r$)r5rfrarqrhlogsqrt)r^rzr{var_avar_bvar_diffprobentropys r5get_uncertaintyz!BradleyTerryModel.get_uncertaintys$c$,,Z84;N;NNPTUUc$,,Z84;N;NNPTUU5=&&z:>%$((4%<00AHTTYIY@Z3ZZ8,,,r4c|jj}|jj}|j}|jr |j nd}|j ||}||z }||z}|j|j|z} |j|j|z} |j|xx|j|| zzzcc<|j|xx|j|| z zz cc<|j|xxdz cc<|j|xxdz cc<|xjdz c_ ||zd|z z} |j|xx| z cc<|j|xx| z cc<| tj|dzzd|z tjd|z dzzz } | |t||j||j|dS)z Update model parameters from a single preference pair. Args: pair: Preference pair to learn from Returns: Dictionary with update metrics rr$rs)loss predictionerrorrorp)rzrMr{rrbrrqrarcr`rdrerfrhrtabs) r^rrzr{rweightpredrgrad reg_grad_a reg_grad_bhessr}s r5updatezBradleyTerryModel.updates__'' __''  $($A$As&&z:>u ~((4??:+FF ((4??:+FF   #t'9'9TJ=N'OO#  #t'9'9TJ=N'OO# Z A%  Z A%  A}D) :&$.& :&$.&v..!e)txxDSXHX?Y1YYZ//*5//*5   r4c ggd}t|D]}|j}|rtj|d}d}|D]=} |j | } || dz }| ddkD} | j dkD} | | k(s9|dz }?|r|t |z nd} |r|t |z nd}|dj| |dj|tjd |dzd | d d |d |S) a( Fit model on a batch of preference pairs. Args: pairs: List of preference pairs epochs: Number of training epochs shuffle: Whether to shuffle pairs each epoch Returns: Dictionary with training metrics over epochs r}accuracyrr}r~rr$rzEpoch z: loss=z.4fz , accuracy=) rangecopyrshufflerrrrrdebug)r^repochsrmetricsepoch epoch_pairs epoch_lossrrresult pred_a_wins actual_a_winsavg_lossrs r5fitzBradleyTerryModel.fit s"2.6] ]E**,K{+JG# !T*fVn, %\2S8 $ S 0 -/qLG !38zCJ.QH/4wU+!H FO " "8 , J  & &x 0 LL6%!)GHS>XVYN[ \1 ]4r4cf|Dcgc]}||j|f}}t|ddScc}w)z Get ranked list of responses by strength. Args: response_ids: List of response IDs to rank Returns: List of (response_id, strength) tuples sorted by strength c |dS)Nr$r3rjs r5r_z/BradleyTerryModel.get_ranking..Cs AaDr4Tkeyreverse)rcsorted)r^ response_idsridrankeds r5 get_rankingzBradleyTerryModel.get_ranking8s;:FF#3,-FFf.$??Gs.c |j|S)z*Get the strength parameter for a response.)rc)r^r2s r5 get_strengthzBradleyTerryModel.get_strengthEs{++r4c|t|jt|jt|j|j|j |j |jdd}t|d5}tj||dddtjd|y#1swY"xYw)zSave model to file.)r`rar^) strengthscounts hessian_diag pair_countconfigwbNzModel saved to ) rurcrdrfrer`rar^rJpicklerKrr)r^rNstaterOs r5savezBradleyTerryModel.saveIsdoo.4<<( !3!34**!%!3!3"&"5"5"&"5"5  (D ! "Q KKq ! " ohZ01 " "s :B22B;c\t|d5}tj|}ddd|d idtfd|d_tt |d_tt|d_|d_ tjd |S#1swYxYw) zLoad model from file.rbNrcjSr[r])modelsr5r_z(BradleyTerryModel.load..as u/C/Cr4rrrrzModel loaded from r3) rJrrTrrcrprdrUrfrerr)rkrNrOrrs @r5rTzBradleyTerryModel.loadZs(D ! #QKKNE #&eHo&&'CU;EWX#Cx9 )%~1FG!,/ ( 34  # #s B""B+N)r{Gz?rT)r`rUrarUr^rUrbrr)rjrUrorU)rzrLr{rLrorU)rrxroDict[str, float])rT)rrrrprrrrozDict[str, List[float]])rzList[ResponseId]rozList[Tuple[ResponseId, float]])r2rLrorUrX)rNrNroz'BradleyTerryModel')r(r)r*r+rrlrqr{rrrrrrvrTr3r4r5rZrZns& # $ #)- III I #' I8666  6&---  -<1 l -#-- - -^ @,2"  r4rZceZdZdZdej dddf d dZd dZ d ddZdd Z dd Z dd Z y)PreferenceLearnera Learns from human preferences using various strategies. Features: - Online, mini-batch, and full batch learning - Multiple model backends (Bradley-Terry, neural) - Curriculum learning support - Learning rate scheduling - Validation and early stopping N rrc|xs t|_||_||_||_||_g|_g|_td|_ d|_ y)aD Initialize preference learner. Args: model: Preference model (creates default if None) strategy: Learning strategy batch_size: Batch size for mini-batch learning validation_split: Fraction for validation patience: Early stopping patience infrN) rZrstrategy batch_sizevalidation_splitpatience_buffer_training_historyrU_best_val_loss_patience_counter)r^rrrrrs r5rzPreferenceLearner.__init__zsT$1/1   $ 0  -/ 9;#El!"r4c^|jtjk(r|jj |S|j j ||jtjk(r=t|j |jk\r|j|j Sy)z Learn from a single preference pair. Args: pair: Preference pair to learn from Returns: Metrics if update occurred, None otherwise N) rr>r?rrrrr@rr _train_batchr#s r5learnzPreferenceLearner.learns~ ==,33 3::$$T* * D! ==,77 74<< DOO3((66r4ctj|tt|d|jz z}|d|}||d}ggdd}t |D]}|j |}|dj||r|j|} |dj| | d|jkr| d|_ d|_ ||d<nI|xjdz c_ |j|jk\rtjd ||S|jj||d|d |r dndd |S) z Learn from a batch of preference pairs. Args: pairs: List of preference pairs epochs: Number of training epochs Returns: Training results r$Nr) train_metrics val_metrics best_epochrrr}rzEarly stopping at epoch r)r train_losstrain_accuracyval_loss)rrrprrrrr _evaluaterrrrrr) r^rr split_idx train_pairs val_pairsresultsr train_result val_results r5 learn_batchzPreferenceLearner.learn_batchsl uE a$*?*?&?@A JY' )*%    6] E,,[9L O $ + +L 9!^^I6  &--j9f%(;(;;*4V* &>ug$FG  " " ) )*62".z":2;Jv. + + 8r4c0d}d}tj||D]C}|jj|}||dz }|ddkD|jdkDk(s?|dz }Eg|_|r|t |z nd|r|t |z ndt |dS)zTrain on a batch of pairs.rrr}r~rr$)r}rn_pairs)rrrrrrr)r^r total_lossrrrs r5rzPreferenceLearner._train_batchs u DZZ&&t,F &. (J|$s* S0@A1     05JU+!05#e*,15z  r4cd}d}|D]}|jj|jj|jj}|j }| t j|dzzd|z t jd|z dzzz }||z }|dkD|dkDk(s|dz }|r|t|z nd|r|t|z dSddS)z,Evaluate on a set of pairs without updating.rrrsr$rr) rrqrzrMr{rrhrtr)r^rrrrrrr}s r5rzPreferenceLearner._evaluates  D::001C1CT__EWEWXDJJE6DHHTE\22a%i488APTHW\L\C]5]]D $ Js  ,1  05JU+!05#e*,  ;<  r4c|jS)zGet training history.)rr]s r5get_training_historyz&PreferenceLearner.get_training_history s%%%r4) rzOptional[BradleyTerryModel]rr>rrprrUrrp)rrxrozOptional[Dict[str, float]])r)rrrrprorX)rrror)rozList[Dict[str, float]]) r(r)r*r+r>r@rrrrrrr3r4r5rrns .2%5%@%@"% #*### #  #  #:07#77  7r ( (&r4rceZdZdZej df d dZ d d dZd dZ d dZ d ddZ y)ActivePreferenceQueryad Selects the most informative comparisons for human annotation. Uses information-theoretic criteria to select comparisons that will most reduce model uncertainty. Features: - Multiple query strategies (uncertainty, information gain, diversity) - Batch query generation - Diversity constraints - Budget-aware selection rcL||_||_||_t|_y)z Initialize active query selector. Args: model: Preference model for uncertainty estimation strategy: Query selection strategy diversity_weight: Weight for diversity in hybrid strategy N)rrdiversity_weightset_queried_pairs)r^rrrs r5rzActivePreferenceQuery.__init__$s$   0BE%r4c~|sgS|Dcgc]Q\}}|j|jf|jvr(|j|jf|jvr||fS}}}|sgSg}|D]+\}}|j||}|j|||f-g}t ||kr |r |j dd|j d}|j|d|df|jj|dj|djf|jdkDrtt|D]f\} \}}}|j|dj|dj|j|j} |||d|j| zz zf|| <ht ||kr|r |Scc}}w)z Select the most informative comparisons. Args: candidates: List of candidate (response_a, response_b) pairs n: Number of queries to select Returns: Selected comparison pairs c |dS)Nr#r3rs r5r_z4ActivePreferenceQuery.select_query..^s adr4Trrr$) rMr_compute_query_scorerrsortpopr$r enumerate_compute_similarity) r^ candidatesrrbscoredscoreselectedbesti similaritys r5 select_queryz"ActivePreferenceQuery.select_query8sI * q!add|4#6#66qtt D$7$77F  I )DAq--a3E MM1a- ( ) (maF KKNDK 9::a=D OOT!Wd1g. /    # #T!WZZa$< =$$q((1&(9Y$A}1e!%!9!9$q'**d1gjjRSRVRVXYX\X\!]J!"AuD4I4IJ4V0V'W XF1IY(maFA sAF9cjtjk(r0jj j j Sjtj k(rjj j j }jjj j }| tj|dzzd|z tjd|z dzzz }||zSjtjk(r\jsyd}jD]9\}}j||j j } t|| };d|z Sjtjk(rjj j j } d} jr%tfdjD}d|z } djz | zj| zzSt!j S)z$Compute query informativeness score.rsr$rc3tK|]/\}}j||jj1ywr[)rrM)rqaqbrrr^s r5rz=ActivePreferenceQuery._compute_query_score..s6%B,,RQTT144@%s58)rrDrErr{rMrFrqrhrtrGrrrrIrr) r^rr uncertaintyryrzmin_similarityrrsimuncertainty_scorediversity_scores ``` r5rz*ActivePreferenceQuery._compute_query_scorems ==M55 5::--addADD9 9 ]]m<< <**44QTT144@K::00qtt.OsD( OO..0B0BC(((s=?)rmodel_agreementunique_responsesr ) rrUrrqrzrMr{rrrr )r^rrrrs r5rz#PreferenceIntegrator.get_statisticsAsT\\" D::001C1CT__EWEWXDs  S 011   u:7>>  >. + ++  +ZKK"K & K* r4r c D Ktdtdtdtdtdd}tdtd }td tdd }td t |t j }tdt|tj}tdtddgdfdgdfg}g}|D]F\}}|D]<} tttj|| } |j| >Htdt|dtdgd} t!dt|dz dD]} | ddD]} |j#|| j$|| || dz}t|| j&d kDrt(j*nt(j,}|j/|| |t1j2d!d"t1j2d#d$%d{\}}|s|s|j5|td&t|d'td(|j7td)t9|}|j;|d*+}td,|d-d.d/d0td1|j=|}|j?|d23}td4t|d5|D][\}}|jA|jB|jB}td6|j&dd d7|j&dd d8|d9d:]td;tE||d<}|jGd*3}tdt|d=|ddD]}td>|d?dd@dA|dBdCd: |j7}tdD|dEd0tdFt!tId2t|dz D]d} || || dz}}|jK|jB|jB}tdG|j&ddHdI|j&ddHdJ|d0ftdKtdLtd|||||dMS7w)Nz+Demonstrate the preference learning system.zF======================================================================z0AIVA Queen - Preference Learning System for RLHFz 1. Initializing components...rr)rrz$ - PreferenceCollector initializedz:memory:rSz" - PreferenceDataset initializedr)r`raz" - BradleyTerryModel initialized)rrz" - PreferenceLearner initializedz& - ActivePreferenceQuery initializedz" 2. Generating sample responses...zWhat is machine learning?)zKMachine learning is a subset of AI that enables systems to learn from data.z!ML is when computers learn stuff.zEMachine learning involves algorithms that improve through experience.zExplain quantum computing.)z>Quantum computing uses qubits that can exist in superposition.z"Quantum computers are really fast.z>Quantum computing leverages quantum mechanics for computation.rz Generated z responsesz 3. Collecting preferences...) annotator_1 annotator_2 annotator_3rr$r#N)rOrzr{rrr<)rr~r|rrz Collected z preference pairsz Statistics: z 4. Training preference model...r)rz Final train accuracy: rr%rz.2%z$ 5. Selecting informative queries...r")rz Selected z queries for annotationz - 'z ...' vs 'z...' (uncertainty: z.3f)z) 6. Integrating with training pipeline...r z training pairsz - Chosen: 'r(z...' (margin: rz.2fz! Model agreement with dataset: r"z% 7. Testing preference predictions...z P('z...' > 'z...'): zG ======================================================================z(Preference Learning System Demo Complete) collectorrUrlearner integrator)&printrrrZrr>r@rrDrErKrNrrrrrrrOrPr!r-r1rruniformr$rrrrrr{rMr rrrq)r-rUrr.query_selectorprompts_and_responses all_responsesrOtextsrPresp annotatorsr annotatorcomp_idr?acceptedrrrrrrrrr/rtpstatsrys r5mainr=[s (O <= (O +,#"%I 01 3G ./ C EE ./e6F6Q6QRG ./*AZAZ[N 23 /0 $'  &(  M.'  'Dtzz|$D   &  '' M#m,-Z 89 *+>J 1c-(1,a 0"#BQ "I11$Q'..(+(Q/2G36mA6F6K6K2Lr2Q%..WiWrWrD#,#>#>%&!>>#s3#)>>"b#9 $?$NHdD D!' "", M#g,'8 9: OI4467 89 -. ME!!%!2G %go&>r&B:&Ns%S TU 1288GJ**:*;H LX'> ?@a1++ADD!$$7  qvvcr{m9QVVCR[M9L[Y\L]]^_`a  67%E7MJ22Q27N M#n-.o >?Ra U r(|CR018 S?QQRSTU  % % 'E -e4E.Fs-K LM 23 3q#m,q01 2LQq1u!51''add3 qvvcr{m8AFF3BK=SzJKL  / 45 (O  ksHR R R "R %I9R __main__)Mr+ __future__rasynciohashlibrloggingrhosrrsyspathrelestio_configrr psycopg2.extrasrrrirabcrr collectionsrr dataclassesr r r r r enumrrpathlibrtypingrrrrrrrrrrrr basicConfigINFO getLoggerrrrNrLr}ryr!r7r>rDrKrxrrrrZrrr r=r(runr3r4r5rRs0#  ;<) #*00(      ,, A   4 5 CL    $d$tD % %  % P J J  J Z / /  / lx x ~ ~~Jyy@a&a&PbbRU U xD zGKKr4