!/il0dZddlZddlZddlZddlmZmZmZmZm Z m Z m Z ddl m Z mZddlmZddlZe GddZe Gdd Zeaej(Z dAdedededededeeefdefdZdd d d d efdddededededededee edfdeeeeegdfdefdZ dBdd d d d efddddeededededededee edfdeeeeegdfdeedefd ZGd!d"Zdefd#ZdCd$Z e!d%k(rddl"Z"e#e"jHd&kDre"jHd&Z%e%d'k(r6e&d(eZ'e'jQD]\Z)Z*e&d)e)d*e*ye%d+k(rHe&d,e&d-da+d.Z,ed/d01d2Z- e-Z.e&d3e.e&d5eye%d6k(rCe&d7e0d/D]/Z1ee1d d89Z2ee1d d 9Z3e&d:e1d&zd*e2d;de%e&d?ye&d@e&d?yy#e$rZ/e&d4e/YdZ/[/dZ/[/wwxYw)Da Genesis Retry Utilities ======================= Exponential backoff with jitter for resilient operations. Usage: from retry_utils import retry, RetryConfig # As decorator @retry(max_attempts=3, base_delay=1.0) def flaky_operation(): # May fail sometimes pass # With custom config config = RetryConfig(max_attempts=5, max_delay=60.0) @retry(config=config) def another_operation(): pass # Programmatic retry result = retry_call(flaky_operation, max_attempts=3) N)CallableAnyOptionalTupleTypeUnionList) dataclassfield)datetimeceZdZUdZdZeed<dZeed<dZ eed<dZ eed <d Z e ed <d Z eeefed <efZeeedfed<dZeeeeegdfed<y) RetryConfigz!Configuration for retry behavior. max_attempts? base_delayN@ max_delay@exponential_baseTjitter?g? jitter_range. exceptionsNon_retry)__name__ __module__ __qualname____doc__rint__annotations__rfloatrrrboolrr Exceptionrrrrr)/mnt/e/genesis-system/core/retry_utils.pyrr#s+L#JIu!e!FD(2L%u %2/8lJd9os*+:BFHhxi 7 =>?Fr'rceZdZUdZdZeed<dZeed<dZeed<dZ eed<dZ eed<dZ e ed <d Z eeed <ee Zeed <dedd fdZdededd fdZdefdZy ) RetryStatsz Statistics for retry operations.r total_callssuccessful_first_trysuccessful_after_retryfailed_all_attempts total_retriestotal_delay_secondsN last_error)default_factoryerrors_by_typeattemptsreturnc|xjdz c_|dk(r|xjdz c_y|xjdz c_|xj|dz z c_y)N)r+r,r-r/)selfr5s r(record_successzRetryStats.record_success<sQ A q=  % % * %  ' '1 , '   (Q, . r'errorc*|xjdz c_|xjdz c_|xj|dz z c_t||_t |j }|jj|ddz|j|<y)Nr8r) r+r.r/strr2typerr4get)r9r5r; error_types r(record_failurezRetryStats.record_failureDs| A   A%  hl*e*%[)) *.*=*=*A*A*a*PST*TJ'r'c n|j|j|j|j|j|j|jzt d|jz |jt d|jz t |jd|j|jd S)Nr8) r+r,r-r.r/ success_ratefirst_try_rater1r4r2) r+r,r-r.r/maxroundr1r4r2r9s r(to_dictzRetryStats.to_dictLs++$($=$=&*&A&A#'#;#;!//!669T9TTq$"2"234"77#aAQAQ:RR#()A)A1#E"11//  r')rrrr r+r!r"r,r-r.r/r1r#r2rr=r dictr4r:r%rArIr&r'r(r*r*0s*K !#!"#C#  M3!$$ $J $ 6ND6/s/t/UsU9UU    r'r*rrrTattemptrrrrrr6c\t|||zz|}|rtj|}||z}|S)a Calculate delay for the given attempt number. Uses exponential backoff with optional jitter: delay = min(base * (exp_base ^ attempt), max_delay) * jitter Args: attempt: Current attempt number (0-indexed) base_delay: Base delay in seconds max_delay: Maximum delay cap exponential_base: Multiplier for backoff jitter: Whether to add randomness jitter_range: Range for jitter multiplier Returns: Delay in seconds )minrandomuniform)rKrrrrrdelayjitter_multipliers r(calculate_delayrRas?4  .'9:I FE"NNL9 "" Lr'rrrrrrrrfuncrr.rc zd} d} t|D]N} ||i| } t5tj| dztxj| z c_ddd| cS| r| y#1swYxYw#|$r}|} | dz|k\rNt5tj | dz|txj| z c_ddd#1swYxYwt | ||||}| |z } |r || dz||td| dzd|d|td|d d tj|Yd}~%d}~wwxYw) az Call a function with retry logic. Args: func: Function to call *args: Positional arguments for func max_attempts: Maximum number of attempts base_delay: Initial delay between retries max_delay: Maximum delay cap exponential_base: Backoff multiplier jitter: Add randomness to delays exceptions: Exception types to catch and retry on_retry: Callback(attempt, exception, delay) on each retry **kwargs: Keyword arguments for func Returns: Result of successful function call Raises: Last exception if all attempts fail Nr0r8rKrrrrz[Retry] Attempt /z failed: z[Retry] Waiting .2fzs before retry...) range _stats_lock _global_statsr:r1rArRprinttimesleep)rTrrrrrrrargskwargslast_exception total_delayrKresulterPs r( retry_callres|B+/NK&'& 4*6*F A,,Wq[911[@1 AM'TK A A  N{l* E!001a@!55D5EE$%#!1 E 5 K1a/ $Wq[M<. !M N $U3K/@A B JJu  9 sGA22A& A2&A/ +A22D:7D53C: D5C A'D55D:)rrrrrrrconfigrfc|rT|j|j|j|j|j|j |j dtdtffd } || |S| S)a  Decorator for retry logic with exponential backoff. Can be used with or without parentheses: @retry def func(): ... @retry(max_attempts=5) def func(): ... @retry(config=RetryConfig(...)) def func(): ... fr6c Rtjfd}|S)Nc 2tg| d|S)NrS)re) r_r`rrrrhrrrrs r(wrapperz)retry..decorator..wrappers?  )%#!1%!   r') functoolswraps) rhrkrrrrrrrs ` r( decoratorzretry..decorators*      r')rrrrrrrr) rTrrrrrrrrfrns ``````` r(retryros4** && $$ !22&& ??X("  r'c eZdZdZ ddededededef dZdZd Z d efd Z dd Z de d d fdZ ed efdZed efdZed ee fdZy ) RetryContextaZ Context manager for retry logic. Usage: with RetryContext(max_attempts=3) as ctx: while ctx.should_continue(): try: result = risky_operation() ctx.success() break except Exception as e: ctx.failed(e) rrrrrct||_||_||_||_||_d|_d|_d|_y)NrF)rrrrr_attempt _succeeded _last_error)r9rrrrrs r(__init__zRetryContext.__init__ s?)$" 0  04r'c|SNr&rHs r( __enter__zRetryContext.__enter__2s r'cy)NFr&)r9exc_typeexc_valexc_tbs r(__exit__zRetryContext.__exit__5sr'r6cR|j|jkxr |j S)z"Check if should continue retrying.)rsrrtrHs r(should_continuezRetryContext.should_continue8s#}}t000H5HHr'Ncd|_y)zMark operation as successful.TNrtrHs r(successzRetryContext.success<s r'r;c||_|xjdz c_|j|jkrt|jdz |j|j |j |j}td|jd|jdtd|ddtj|y y ) z%Record failure and wait before retry.r8rVz[RetryContext] Attempt rWz failedz[RetryContext] Waiting rXzs...N) rursrrRrrrrr\r]r^)r9r;rPs r(failedzRetryContext.failed@s   ==4,, ,# )??..!%!6!6{{ E +DMM?!Dzl*KLL!r'z Result: z Failed: z Stats: delayszDelay progression (5 attempts):F)rrz Attempt z.1fzs (with jitter: ~zs)zUnknown command: z0Usage: python retry_utils.py [stats|demo|delays]zGenesis Retry Utilities)rrrTrrxr)4r r]rNrltypingrrrrrrr dataclassesr r r threadingrr*r[LockrZr!r#r$rRr%rerorqrJrrrsyslenargvcmdr\ritemskvrrrrcrdrYirPdelay_jr&r'r(rs2 DDD(  G G  G ( (  ( X inn !(2 ! !!! !  ! u % ! !N!/8lBFO OO O  O  O Od9os*+Oxi 7 =>?O Of $7!/8lBF$(7 8 77 7  7  7 7d9os*+7xi 7 =>?7 [ !77tM M `'' % z 388}qhhqk '> ' (#%E  %11#Rsm$ %F] 8 9 (OJI c 2 "3 " ((* 6(+, Io/01 2 H_ 3 41X W'c%H)!DI 1Q3%r%4Egc]RTUV W %cU+ , D E '( @A_< ( 1#&'' (sH//I 4 II