/is ddlmZddlZddlZddlZddlmZmZddlm Z m Z m Z m Z m Z ddlmZmZmZddlZddlmZmZddlmZddlmZdd lmZmZmZddlZddlZd d lm Z d d l!m"Z"d d l#m$Z$d dl%m&Z&m'Z'm(Z(d dl)m*Z*d dl+m,Z,d dl-m.Z.d dl/m0Z0d dl1m2Z2m3Z3m4Z4ddl5Z5ddl6Z6ddl7Z7ddl8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@GddeZAGddeAZBGddeCZDGddeDZEGd d!eDZFGd"d#eAZGy)$) annotationsN)ABCabstractmethod)CallableDictAnyListUnion)OptionalAsyncGeneratorFinal)PageError) TimeoutError)BytesIO)Image ImageDraw ImageFont)load_js_scriptAsyncCrawlResponse)SCREENSHOT_HEIGHT_TRESHOLD) BrowserConfigCrawlerRunConfigHTTPCrawlerConfig) AsyncLogger)SSLCertificate)ValidUAGenerator)BrowserManager)BrowserAdapterPlaywrightAdapterUndetectedAdapter) ClientTimeout)urlparse)MappingProxyType)partialc"eZdZdZeddZy)AsyncCrawlerStrategyze Abstract base class for crawler strategies. Subclasses must implement the crawl method. c KywN)selfurlkwargss [/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/crawl4ai/async_crawler_strategy.pycrawlzAsyncCrawlerStrategy.crawl)s  sN)r.strreturnr)__name__ __module__ __qualname____doc__rr1r,r0r)r)#s   r8r)ceZdZdZ d' d(dZdZdZdZdZd)dZ d*d Z d+d Z d,d Z d-d Z d.d/d Z d. d0dZdZd1dZ d2dZ d2dZd3d4dZd5dZdZd6dZd7dZd8dZ d9dZ d: d;dZd1dZddZ d?d Z! d?d!Z"d"Z#d@dAd#Z$dBd$Z%dCd%Z&dDd&Z'y)EAsyncPlaywrightCrawlerStrategya Crawler strategy using Playwright. Attributes: browser_config (BrowserConfig): Configuration object containing browser settings. logger (AsyncLogger): Logger instance for recording events and errors. _downloaded_files (List[str]): List of downloaded file paths. hooks (Dict[str, Callable]): Dictionary of hooks for custom behavior. browser_manager (BrowserManager): Manager for browser creation and management. Methods: __init__(self, browser_config=None, logger=None, **kwargs): Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration. __aenter__(self): Start the browser and initialize the browser manager. __aexit__(self, exc_type, exc_val, exc_tb): Close the browser and clean up resources. start(self): Start the browser and initialize the browser manager. close(self): Close the browser and clean up resources. kill_session(self, session_id): Kill a browser session and clean up resources. crawl(self, url, **kwargs): Run the crawler for a single URL. Nc "|xstj||_||_|xs t |_g|_dddddddddd |_t|j|jt|j t|_ y)a Initialize the AsyncPlaywrightCrawlerStrategy with a browser configuration. Args: browser_config (BrowserConfig): Configuration object containing browser settings. If None, will be created from kwargs for backwards compatibility. logger: Logger instance for recording events and errors. browser_adapter (BrowserAdapter): Browser adapter for handling browser-specific operations. If None, defaults to PlaywrightAdapter. **kwargs: Additional arguments for backwards compatibility and extending functionality. N) on_browser_createdon_page_context_createdon_user_agent_updatedon_execution_startedon_execution_ended before_goto after_gotobefore_return_htmlbefore_retrieve_html)browser_configloggeruse_undetected) r from_kwargsrErFr"adapter_downloaded_fileshooksr isinstancer#browser_manager)r-rErFbrowser_adapterr/s r0__init__z'AsyncPlaywrightCrawlerStrategy.__init__Js-Q 0I0I&0Q '=*;*= "$#''+%)$("&"&$(    ...;;%dll4EF r8cBK|jd{|S7wr+startr-s r0 __aenter__z)AsyncPlaywrightCrawlerStrategy.__aenter__vjjl   c@K|jd{y7wr+closer-exc_typeexc_valexc_tbs r0 __aexit__z(AsyncPlaywrightCrawlerStrategy.__aexit__zjjl cK|jjd{|jd|jj|jjd{y7I7w)zG Start the browser and initialize the browser manager. Nr<)context)rMrR execute_hookbrowserdefault_contextrSs r0rRz$AsyncPlaywrightCrawlerStrategy.start}si""((***  ( (((88     + s"A.A*AA.$A,%A.,A.cjK|jjd{dt_y7w)z; Close the browser and clean up resources. N)rMrYr _playwright_instancerSs r0rYz$AsyncPlaywrightCrawlerStrategy.closes-""((***.2+ +s 313cK|jjdd|jj|d{y7w)z Kill a browser session and clean up resources. Args: session_id (str): The ID of the session to kill. Returns: None zSSession auto-kill is enabled in the new version. No need to manually kill sessions.WARNINGmessagetagN)rFwarningrM kill_session)r- session_ids r0rnz+AsyncPlaywrightCrawlerStrategy.kill_sessionsA i  ""// ;;;s? ?r8cK|jj|}|r-tj|r||i|d{S||i|S|r|dSdS7w)a4 Execute a hook function for a specific hook type. Args: hook_type (str): The type of the hook. *args: Variable length positional arguments. **kwargs: Keyword arguments. Returns: The return value of the hook function, if any. Nr)rKgetasyncioiscoroutinefunction)r-rsargsr/rts r0rcz+AsyncPlaywrightCrawlerStrategy.execute_hooksdzz~~i( **40!426222T,V,,tAw(D(3s>AAAc||_y)z Update the user agent for the browser. Args: user_agent (str): The new user agent string. Returns: None N) user_agent)r-r|s r0update_user_agentz0AsyncPlaywrightCrawlerStrategy.update_user_agents %r8c||_y)z Set custom headers for the browser. Args: headers (Dict[str, str]): A dictionary of headers to set. Returns: None N)headers)r-rs r0set_custom_headersz1AsyncPlaywrightCrawlerStrategy.set_custom_headerss  r8cPK|j}|jdr.|ddj}|j|||d{S|jdr0|ddj} |j||d{y|jd s|jd r|j|||d{S |j||d{y77a#t$r3}dt |vrt d|d |d td |d d}~wwxYw7d7J#t$rf}dt |vrt d|d |d  |j|d|d|d{7cYd}~S#t$rtd|dwxYwd}~wwxYww)a Wait for a condition in a smart way. This functions works as below: 1. If wait_for starts with 'js:', it assumes it's a JavaScript function and waits for it to return true. 2. If wait_for starts with 'css:', it assumes it's a CSS selector and waits for it to be present. 3. Otherwise, it tries to evaluate wait_for as a JavaScript function and waits for it to return true. 4. If it's not a JavaScript function, it assumes it's a CSS selector and waits for it to be present. This is a more advanced version of the wait_for parameter in CrawlerStrategy.crawl(). Args: page: Playwright page object wait_for (str): The condition to wait for. Can be a CSS selector, a JavaScript function, or explicitly prefixed with 'js:' or 'css:'. timeout (float): Maximum time to wait in milliseconds Returns: None zjs:Nzcss:timeoutTimeoutzTimeout after zms waiting for selector ''zInvalid CSS selector: 'z()functionz() => {}zInvalid wait_for parameter: 'zp'. It should be either a valid CSS selector, a JavaScript function, or explicitly prefixed with 'js:' or 'css:'.)strip startswithcsp_compliant_waitwait_for_selectorrr2rrr)r-pagewait_forrjs_code css_selectores r0 smart_waitz)AsyncPlaywrightCrawlerStrategy.smart_waits$>>#   u %qrl((*G00wHH H   (#AB<--/L P,,\7,KKK""4(H,?,? ,K!44T8WMMM0070KKK+I L PA&&( 1J<.XYZ%'>|nA%NOO  PNL CF**,WI5NxjXYZ )-)@)@ $ "&=w*$$ %","?zJO!O#sA F& C-(F&7C1 C/C19F& D0 F&D4'D2(D4,F&/C11 D-:.D((D--F&2D44 F#=FF7E:8F<F#=F&FFF##F&cKd|d|d} |jj||d{}|S7#t$r.}dt|vrt dt|Yd}~yd}~wwxYww)a Wait for a condition in a CSP-compliant way. Args: page: Playwright page object user_wait_function: JavaScript function as string that returns boolean timeout: Maximum time to wait in milliseconds Returns: bool: True if condition was met, False if timed out Raises: RuntimeError: If there's an error evaluating the condition z8 async () => { const userFunction = z; const startTime = Date.now(); try { while (true) { if (await userFunction()) { return true; } if (Date.now() - startTime > aR) { return false; // Return false instead of throwing } await new Promise(resolve => setTimeout(resolve, 100)); } } catch (error) { throw new Error(`Error evaluating condition: ${error.message}`); } } NzError evaluating conditionz#Failed to evaluate wait condition: F)rIevaluate Exceptionr2 RuntimeError)r-ruser_wait_functionr wrapper_jsresultrs r0rz1AsyncPlaywrightCrawlerStrategy.csp_compliant_wait's"""4!5623: ;  ( <<00zBBFMC +s1v5"%HQ#QRR  s6 A/535A/5 A,$A'"A/'A,,A/c K|jdd{}t|D]\}} |jd|dd{|jd{}|rz|j ddd{|jdd{}d |}|j d d }|j j|d |d |d|dd{n |jjddd|i|S77777v73#t$r5} |jjdd|t| dYd} ~ !d} ~ wwxYww)a  Process iframes on a page. This function will extract the content of each iframe and replace it with a div containing the extracted content. Args: page: Playwright page object Returns: Playwright page object iframeNz"(element) => element.id = "iframe-"load0urz() => document.body.innerHTMLzextracted-iframe-content-`z\`zl () => { const iframe = document.getElementById('iframe-zx'); const div = document.createElement('div'); div.innerHTML = `z0`; div.className = 'zf'; iframe.replaceWith(div); } z1Could not access content frame for iframe {index}SCRAPEindexrkrlparamsz(Error processing iframe {index}: {error}ERROR)rerror) query_selector_all enumerater content_framewait_for_load_statereplacerIrFrmrrr2) r-riframesirframeiframe_content class_name_iframers r0process_iframesz.AsyncPlaywrightCrawlerStrategy.process_iframesUs//99"7+. IAv- oo(J1#Q&OPPP%2244334 ,1>>7,&N $=QC!@J-44S%@G,,//LLM3O..5Y7..8\:    KK'' S$ '|(I. b g: Q5 & "  !!F%&Q8" sEDEDDDDD;D <DD ADD %D?EDD D D D E *EEE  EcBK|jd{|jdxsttj}|jd|j }|j jtd||d|d{\}}|S77 w)au Creates a new browser session and returns its ID. A browse session is a unique openned page can be reused for multiple crawls. This function is asynchronous and returns a string representing the session ID. Args: **kwargs: Optional keyword arguments to configure the session. Returns: str: The session ID. Nror|)ror|r,) rRrwr2uuiduuid4r|rMget_pager)r-r/ror|rrbs r0create_sessionz-AsyncPlaywrightCrawlerStrategy.create_sessionsjjlZZ -BTZZ\1B ZZ doo> "22;;>A Bf55 5 ^^I &#..*BcnnU[F\)) 0!! 0  0$$ 0 0  0 %% 0.. 0$$ 0  0&& 0// 0//  "__S&999~~i("%ab'ww~~o6+.D_DU,VWW/3A$Q668D$$#&..":s12wAB%!1'$( Q _6.:$$s?AHG4 DH G7 AH"G93AH7H9H>Hc"56K|_i}d}d}}g_g5g}|j}|r|j_nR|js|j dk(r7t jdhi|jxsij_jj|d{\6} |js|js |jr"| jtdd{jd6| |d{|j rH5fd} 5fd} 5fd } 6j#d | 6j#d | 6j#d | d} d}|j$rHj&j)6|d{} j&j+6|d{} d}|j,rt/j0}jj2r6j#d fd|j4s\jd6| |d{j7dxs$j7dxsj7d}|rÉj7drZdd}t8j:j=|st?d|tA|dd5}|jC}dddnj7drddndd}6jE|jFd{d}|jHxs}d}i}n" |jJjMddrUtOjPt9jRdjU}6jWd d!|d"id{6jY|jF|jZ#d{}6j}|d}i}nm|}|jf}|rE|jhr9|jh}|jkd{}|r|}|}|r |jhr9|jl}|jn}jd*6| ||+d{nd}i} 6jqd,d-d./d{js6d0d.1d{}|s3|jts'jw6d{}t]d2|jj|s|j~s |jr}6jd6d{tjd7d{js6d8d91d{}|s)j`rj`jd:d;<jj|s|jr j6d{}|d=}|d>} jj}!t|!| z|z d?z}"6j|!|"d@d{t|!| z |"|z }#6jj6d{}$|$jdA| |dBd|#dCd{|jr/j6|j|jd{|jr$j6|jd{|jrj6|jd{}|dGs/j`jdHdIdF|jMdFi(jdJ6| |d{jdK6| ||Ld{|js |jr6jjdMdMd{6jjd{6jjd{6jjdNd{|js |jr |jrK |j |jn |jZ}%j6|j|%1d{jj|sLtdP}& 6jd6dQ1d{j&j6|&d{|jrj6d{6jdT6| |d{|jr'tj|jd{|jrj6d{|jr |jjdUD'cgc]}'|'j}(}'g})|(D]<}* j&j6dV|*dWd{}+|)j|+>dZd[j|)zd\z},n6jd{},jd^6|,| |_d{tj}-d}.d}/d}0|jrj6d{}.|jrj6d{}0|jrX|jr'tj|jd{j6|j`d{}/|/s|.s|0r5j`jcdadbdctj|-z i(didj6fdd }1|j$rJtj&der4j&j6d{}2|j|26j}t|,||||/|.|0|1|jr jnd||j r5nd|j$r|ndf 6jjj}3tdg|3D}4|jrS|4dBkr-jjsjjrS|j r66jd  6jd  6jd  |j$rptj&der4j&j6d{}2|j|2j&j6| |d{6jd{S7 7 v7 \7 7 7 ?#1swY xYw7 k7 7 #t\$rk}d$t_|vr<jj2r&j`jcd%d&d'i(d}nted)t_|Yd}~ d}~wwxYw7 7 7 v7 ]7 9#t\$rnjw6d{7}jjxr j`j{d3d4d5|i(|jtst]d2|Y wxYw7 a7 G7 .77}7K7.#t$r4}j`jdDdEdFt_|i(Yd}~dd}~wwxYw767 777s7:7777o#t$r}tedOt_|d}~wwxYw7\#t$rYfwxYw7K#t$r4}j`jdRdSdFt_|i(Yd}~d}~wwxYw7i7O77cc}'w7#t\$r%}tdX|*dYt_|Yd}~d}~wwxYw#t\$r}ted]t_|d}~wwxYw777T72777N777#t$r}|d}~wwxYw#6jjj}3tdg|3D}4|jrw|4dBkr-jjsjjrw|j r66jd  6jd  6jd  |j$rrtj&der5j&j6d{7}2|j|2j&j6| |d{76jd{7wxYww)ka Internal method to crawl web URLs with the specified configuration. Includes optional network and console capturing. Args: url (str): The web URL to crawl config (CrawlerRunConfig): Configuration object controlling the crawl behavior Returns: AsyncCrawlResponse: The response containing HTML, headers, status code, and optional data NrandomcrawlerRunConfignavigator_overriderr=)rbrc K d} |j}|r |jdd}j d|j |jt|j||j|jtjdy#t$rdt|d}YwxYw#t$rd}YwxYw#t$r~}jr-jjd |j d |d j d |j t|tjdYd}~yd}~wwxYww)Nrrerrorsz[Binary data: z bytes]z[Error retrieving post data]request) event_typer.methodr post_data resource_typeis_navigation_request timestampz$Error capturing request details for : CAPTURErlrequest_capture_errorrr.rr)post_data_bufferdecodeUnicodeDecodeErrorlenrappendr.rdictrrrtimerFrmr2)r post_data_strrrcaptured_requestsr-s r0handle_request_capturezIAsyncPlaywrightCrawlerStrategy._crawl_web..handle_request_capture/sgU$(M G$+$<$< $Z1:1A1A'R[1A1\ &,,&/&{{")..#'#8%2)0)>)>181N1N1P%)YY[ .  %7Z3A#i.AQQX1YZ$G(F G!U{{ ++.RSZS^S^R__abcad,ekt+u%,,.handle_response_captureMs<W$+3--/$9 &,,&0'||"*//'/';';#'(8(8#9/7/K/K*2*:*:*A*A%)YY[#I". %:$$#$&!W{{ ++.ST\T`T`Saacdecf,gmv+w%,,B7EB B4(B/*B7/B44B77 D>A4D94E9D>>Ec K jd|j|j|j|jrt |jndt j dy#t$r~}jr-jjd|jd|djd|jt |t j d Yd}~yd}~wwxYww) Nrequest_failedzUnknown failure)rr.rr failure_textrz+Error capturing request failed details for rrrrequest_failed_capture_errorr) rr.rrfailurer2rrrFrm)rrrr-s r0handle_request_failed_capturezPAsyncPlaywrightCrawlerStrategy._crawl_web..handle_request_failed_captureks \%,,&6&{{")..)0)>)>@GGOO(z;AsyncPlaywrightCrawlerStrategy._crawl_web..s W%8%8--h7&r8rA)rbr.rrrrrrrrrrr wait_untilr use_csp_nonceF zContent-Security-Policyz-default-src 'self'; script-src 'self' 'nonce-z' 'strict-dynamic')rrznet::ERR_ABORTEDz1Navigation aborted, likely due to file download: GOTOr.rzFailed on navigating ACS-GOTO: rB)rbr.rrrattachedr)statera() => { const element = document.body; if (!element) return false; const style = window.getComputedStyle(element); const isVisible = style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; return isVisible; }rzBody element is hidden: zBody visibility info: {info}DEBUGinfodomcontentloaded皙?zQ() => Array.from(document.getElementsByTagName('img')).every(img => img.complete)z)Some images failed to load within timeoutrrjheightwidthgffffff?r$r#z"Emulation.setDeviceMetricsOverrider)r$r#deviceScaleFactormobilescalez-Failed to adjust viewport to content: {error}VIEWPORTrsuccessz)User script execution had issues: {error}JS_EXECr?r@)rbrrd ArrowDownzWait condition failed: update_image_dimensionsz(Error updating image dimensions: {error}rrD,z&Array.from(document.querySelectorAll("zn")) .map(el => el.outerHTML) .join('')z-Warning: Could not get content for selector 'z': z
 z
z Failed to extract HTML content: rC)rrrbrscreenshot_height_thresholdz;Exporting media (PDF/MHTML/screenshot) took {duration:.2f}sEXPORTdurationcKjjdd|dtj|d{j d{S77w)Nz?Waiting for {delay} seconds before retrieving content for {url}INFO)delayr.r)rFrrxsleepcontent)r8rr-r.s r0rzFAsyncPlaywrightCrawlerStrategy._crawl_web..get_delayed_contentsX   ]%*37! mmE***!\\^++++s!:AAAAAAretrieve_console_messages) rrjs_execution_resultrrrrrssl_certificatedownloaded_filesredirected_urlnetwork_requestsconsole_messagesc3FK|]}t|jywr+)rpages).0rbs r0 z.2sMWc'--0Ms!r,)@)r8floatr3r2)r.rJr|rEruser_agent_modergenerateuser_agent_generator_configrMroverride_navigatorradd_init_scriptrrcronrrIsetup_console_capturesetup_error_capturefetch_ssl_certificaterfrom_urlaccept_downloadsjs_onlyrrrrrrr set_contentrbase_url experimentalrwhashlibsha256urandom hexdigestset_extra_http_headersgoto page_timeoutrr2rFrrrredirected_fromrrrrrignore_body_visibilitycheck_visibilityverbosedebug text_modewait_for_imagesadjust_viewport_to_contentrrxr9rmget_page_dimensionsviewport_widthintset_viewport_sizeminrbnew_cdp_sessionsendrr_handle_full_page_scan scroll_delaymax_scroll_stepsvirtual_scroll_config_handle_virtual_scrollrrobust_execute_user_scriptmousemovedownupkeyboardpressrrwait_for_timeoutrPlaywrightTimeoutErrorrrrdelay_before_return_htmlrsplitrrprintjoinr:r perf_counterr export_pdfrrscreenshot_wait_fortake_screenshotr3hasattrr;extendrrdcontextssumrouse_managed_browserheadlessremove_listenercleanup_console_capturerY)7r-r.rrexecution_resultrr?captured_consoleuser_agent_to_overriderbrr rhandle_console handle_errorssl_certis_local_contentrr html_contentrnoncer first_respreqprev_req prev_resp is_visiblevisibility_info images_loaded dimensions page_height page_width target_width target_heightr(cdprupdate_image_dimensions_jss selectors html_partsselectorr:rstart_export_timerrrrfinal_messages all_contexts total_pagesrrs7`` @@r0rz)AsyncPlaywrightCrawlerStrategy._crawl_webs  "$"(!2!2 !-CD   * \\V33x?-H-=-?-H-H.55;.D   * #22;;V;TT g  $ $(<(< )).9N*OP P P 94Y_```  * * U< W< \ GGI5 6 GGJ 7 8 GGO%B C  * *#'<<#E#EdL\#]]N!%!A!A$HX!YYL } #H++)2237""33>>'' tWRU^d'eee$'>>)#<#rx@X#r\_\j\jkq\r #~~i0*-ab'!ww~~o>"36L_L]4^"__!/3I4Q+,668L4436..2Js12wPSTUTVPW **$B$L$L$NE#'"="=$=AnotnuvH@I!"# *.F,=,=vGZGZ*3*$*.( '&) +-(%- &..!c&9&9'*':':H.6.?.?.A(AI(-6 "*C "c&9&9'1&7&7 +5+=+=('' $S8\b( " #% ! N,,V:u,UUU$(#:#:"$; $  "&*G*G,0,A,A$,G&GO":?:K LMMt&&00&&&*K*K../ABBBmmC((('+&=&=g '>'! %KK'' K$( &&00V5V5V"'+'?'?'E!EJ",X"6K!+G!4J$(#6#6#E#EL$' z(AK(ORV(V$WM00".-H z 9=;;VWE $ < g^d'eee''(O(OP! , ,..74<>t^Uabbbjjl""UU Q ap^Y.f44 W $! \.Q7D# &8& 44":?:K LMM5 NvC(!"F C !KK'' O& 'Q0(fV$f|0'%6!K&)@Q'IJJKV1Q KK%% J# 'Q0&8 bD9 !T ' %i!$QRZQ[[^_bcd_e^f"ghhi T&)I#a&'RSST, 7<D#."Tb*\c#7 G   <<//88LM MMK  !t':':'N'NRVReReRnRn22((4JK((5LM((:WX22t||-HI/3||/U/UVZ/[)[)[(//?,,>>t^Uabbbjjl""sB,AE52u.3AAE5;u1<AE5u4BAE5 u7!#AE5u:AE5 A4A@>u=?BA@ vAA@"v #A@A,v,v-/vvv-AA@/x 0A@ 2A@;x< A@xxx9x:&x x!x3AA@5z6A@zA@.z/AA@z+zAz+%z"&4z+z%z+:z(;z+?6A@5{+6/A@%{.&/A@{1AA@&{4'A@{7;A@{:!A@"{=#!A@|"A@'|()A@A| || !A@?|4|1|4}9}:}> A@~A@;~ <2A@.~ /$A@~A@% ~  ##~~~ 5A@ 4 A@'7(>A@&:'$A@ = >A@ A@ $A@/A@0BA@8A@9A,A@%C3AE5A@ 6AE5A@ AE5(A@)AE51AE54AE57AE5:AE5=A@v  A@vv x A x?A@x  A@A@xxxz9x<:AzA@zA@A@A@z+"z+%z+(z++ {(4){#A@#{((A@.A@1A@4A@7A@:A@=A@A@A@| |.|))|..A@1|44 }=}}} ~)}?9A@?~A@ A@ A@A@ ~  $>    1,,11A@7A@:A@=A@@A@@A@@A@@ AE5@ AE5@AE5@ A@"@A@@A@"@"A@%@%C4AE2DADD7AE2EAEEAE2E+AE.E,AE2E2AE5c nK |j}|P|j|jj|jjdd{|j}|j d|jj}|}|j |d||d{|j|d{}|d}d} ||krd|| |k\rn\t||z|}|j |d||d{| dz } |j|d{}|d} | |kDr| }||krd|j |ddd{|j |d|d{y7777r7V7+7#t$r3} |jjddd t| i Yd} ~ yd} ~ wwxYww) au Helper method to handle full page scanning. How it works: 1. Get the viewport height. 2. Scroll to the bottom of the page. 3. Get the total height of the page. 4. Scroll back to the top of the page. 5. Scroll to the bottom of the page again. 6. Continue scrolling until the bottom of the page is reached. Args: page (Page): The Playwright page object scroll_delay (float): The delay between page scrolls max_scroll_steps (Optional[int]): Maximum number of scroll steps to perform. If None, scrolls until end. Nr%r#r)r8rz)Failed to perform full page scan: {error} PAGE_SCANrr) viewport_sizerirErgviewport_heightrw safe_scrollrfrjrrFrmr2) r-rrnrorrcurrent_positionr total_heightscroll_step_count new_heightrs r0rmz5AsyncPlaywrightCrawlerStrategy._handle_full_page_scanJs$: : ..M$,,"11@@DL_L_LoLop!% 2 2 +//$--==O / ""4,VB / :  KK  CQ(    sF5A E6E'AE6"E*#E6;E, { const container = document.querySelector(config.container_selector); if (!container) { throw new Error(`Container not found: ${config.container_selector}`); } // List to store HTML chunks when content is replaced const htmlChunks = []; let previousHTML = container.innerHTML; let scrollCount = 0; // Determine scroll amount let scrollAmount; if (typeof config.scroll_by === 'number') { scrollAmount = config.scroll_by; } else if (config.scroll_by === 'page_height') { scrollAmount = window.innerHeight; } else { // container_height scrollAmount = container.offsetHeight; } // Perform scrolling while (scrollCount < config.scroll_count) { // Scroll the container container.scrollTop += scrollAmount; // Wait for content to potentially load await new Promise(resolve => setTimeout(resolve, config.wait_after_scroll * 1000)); // Get current HTML const currentHTML = container.innerHTML; // Determine what changed if (currentHTML === previousHTML) { // Case 0: No change - continue scrolling console.log(`Scroll ${scrollCount + 1}: No change in content`); } else if (currentHTML.startsWith(previousHTML)) { // Case 1: New items appended - content already in page console.log(`Scroll ${scrollCount + 1}: New items appended`); } else { // Case 2: Items replaced - capture the previous HTML console.log(`Scroll ${scrollCount + 1}: Content replaced, capturing chunk`); htmlChunks.push(previousHTML); } // Update previous HTML for next iteration previousHTML = currentHTML; scrollCount++; // Check if we've reached the end if (container.scrollTop + container.clientHeight >= container.scrollHeight - 10) { console.log(`Reached end of scrollable content at scroll ${scrollCount}`); // Capture final chunk if content was replaced if (htmlChunks.length > 0) { htmlChunks.push(currentHTML); } break; } } // If we have chunks (case 2 occurred), merge them if (htmlChunks.length > 0) { console.log(`Merging ${htmlChunks.length} HTML chunks`); // Parse all chunks to extract unique elements const tempDiv = document.createElement('div'); const seenTexts = new Set(); const uniqueElements = []; // Process each chunk for (const chunk of htmlChunks) { tempDiv.innerHTML = chunk; const elements = tempDiv.children; for (let i = 0; i < elements.length; i++) { const element = elements[i]; // Normalize text for deduplication const normalizedText = element.innerText .toLowerCase() .replace(/[\s\W]/g, ''); // Remove spaces and symbols if (!seenTexts.has(normalizedText)) { seenTexts.add(normalizedText); uniqueElements.push(element.outerHTML); } } } // Replace container content with merged unique elements container.innerHTML = uniqueElements.join('\n'); console.log(`Merged ${uniqueElements.length} unique elements from ${htmlChunks.length} chunks`); return { success: true, chunksCount: htmlChunks.length, uniqueCount: uniqueElements.length, replaced: true }; } else { console.log('No content replacement detected, all content remains in page'); return { success: true, chunksCount: 0, uniqueCount: 0, replaced: false }; } } NreplacedFzNVirtual scroll completed. Merged {unique} unique elements from {chunks} chunks uniqueCountr chunksCount)uniquechunkszAVirtual scroll completed. Content was appended, no merging neededrjz&Virtual scroll capture failed: {error}r) async_configsrrLr from_dictrFrcontainer_selectorrIrto_dictrwr*rrr2)r-rrrvirtual_scroll_jsrrs r0rqz5AsyncPlaywrightCrawlerStrategy._handle_virtual_scrolls5&U  :&$',66v> KK  S"F$=$=>  m! ` <<007H&..JZ[[Fzz*e, ##l!"(**]A">"(**]A">$   _!!\"  KK  @Q(    sND=BC>C<AC>D=C>;D=<C>> D:)D50D=5D::D=c \K |j}tjj|jj |}|j jdd||dtj}|j|d{tj}|jj||j jdd||||z dd d y7]#t$r3}|j jd d d t!|iYd}~yd}~wwxYww)az Handle file downloads. How it works: 1. Get the suggested filename. 2. Get the download path. 3. Log the download. 4. Start the download. 5. Save the downloaded file. 6. Log the completion. Args: download (Download): The Playwright download object Returns: None z Downloading {filename} to {path}FETCH)filenamerrNz"Downloaded {filename} successfullyCOMPLETEz.2fr)rrr5z"Failed to handle download: {error}rr)suggested_filenamerrr~rEdownloads_pathrFrrrsave_asrJrr*rrr2)r-rr download_path start_timeend_timers r0rz/AsyncPlaywrightCrawlerStrategy._handle_downloadCs!$ !)! { try { const removeOverlays = a; await removeOverlays(); return { success: true }; } catch (error) { return { success: false, error: error.toString(), stack: error.stack }; } })() Niz*Failed to remove overlay elements: {error}rrr)rrIrryrrFrmr2)r-rremove_overlays_jsrs r0rz6AsyncPlaywrightCrawlerStrategy.remove_overlay_elementsts,,EF ,,''01C/C D   "'', , ,# " -  KK  DQ(    sP B#AAA A ABAA B)B B BBcFK|jdd{}|S7w)z Exports the current page as a PDF. Args: page (Page): The Playwright page object Returns: bytes: The PDF data T)print_backgroundN)r)r-rrs r0rz)AsyncPlaywrightCrawlerStrategy.export_pdfs%4889s !!c K |jddd{|jddd{|jdd{|jdd{|jj|d{}|jd d did{}|jd}|jd{|S7777t#t$r?}|jr)|jj dd d t |i Yd}~d}~wwxYw777]#t$r?}|jr)|jjdd d t |i Yd}~yd}~wwxYww)a Captures the current page as MHTML using CDP. MHTML (MIME HTML) is a web page archive format that combines the HTML content with its resources (images, CSS, etc.) into a single MIME-encoded file. Args: page (Page): The Playwright page object Returns: Optional[str]: The MHTML content as a string, or None if there was an error r rN networkidler"a () => new Promise(resolve => { // First requestAnimationFrame gets scheduled after the next repaint requestAnimationFrame(() => { // Second requestAnimationFrame gets called after all animations complete requestAnimationFrame(resolve); }); }) z&Wait for load state timed out: {error}MHTMLrrzPage.captureSnapshotformatmhtmldataz Failed to capture MHTML: {error})rryrrrFrmr2rbrkrlrwdetachrr)r-rr cdp_sessionr mhtml_contents r0rz,AsyncPlaywrightCrawlerStrategy.capture_mhtmls1  ../A4.PPP..}d.KKK++D111mm%$!% < #SV,"  sE;CCCCCCC(C)C-D0 D* D0'D,((D0D.D0E;CCCC D'(5D"D0"D''D0,D0.D00 E895E3.E;3E88E;c|Kgfd}|jd||j|d{S7w)z Captures console messages from the page. Args: page (Page): The Playwright page object Returns: List[Dict[str, Union[str, float]]]: A list of captured console messages c |j}|j}||tjd}j|y#t$r;}j r%j j d|dYd}~yYd}~yd}~wwxYw)N)typerrz!Error capturing console message: rr)rrrrrrFrm)msg message_type message_textentryrrr-s r0handle_console_messagezXAsyncPlaywrightCrawlerStrategy._capture_console_messages..handle_console_messages "xx "xx )(!% !''. ;;KK'';A3?Y( sAA B ,BB consoleN)rMr\)r-r file_pathrrs` @r0_capture_console_messagesz8AsyncPlaywrightCrawlerStrategy._capture_console_messagessB "  12ii """ #s /<:<cNKd}d}d}d} |xs t}|jj|d{\}}|j|dd{|jr|j |d{}|j r|j |d{}|jr[|jr'tj|jd{t|dd}|j||d{}|||f|r |jd{SS77777U7/7#t$rYSwxYw#t$r} dt| } |j j#dd d | i |r|jrt%j&d d d} t)j*| } t-j.} | j1d| d| t3}| j5|dt7j8|j;j=d}|||fcYd} ~ |r* |jd{7S#t$rYSwxYwSd} ~ wwxYw#|r* |jd{7w#t$rYwwxYwwxYww)a Generate media (screenshot, PDF, MHTML) from raw HTML content. This method is used for raw: and file:// URLs where we have HTML content but need to render it in a browser to generate media outputs. Args: html (str): The raw HTML content to render config (CrawlerRunConfig, optional): Configuration for media options Returns: tuple: (screenshot_data, pdf_data, mhtml_data) - any can be None Nrr rr3r2z$Failed to generate media from HTML: z%HTML media generation failed: {error}rrrRGBi iXblackcolor rrrfillfontJPEGrr)rrMrrTrrrrrrxr9getattrrrYrr2rFrrnewrDrawr load_defaultrrsavebase64 b64encodegetvaluer)r-rrrrrrrbr3r error_messageimgdrawrbuffereds r0_generate_media_from_htmlz8AsyncPlaywrightCrawlerStrategy._generate_media_from_htmls  0 1/1F"&"6"6"?"?QW"?"XXMD'""44F"G G Gzz!%!66###'#5#5d#;;   --!--(B(BCCC.5f>[]a.b+(,(<(<6Q)=)##Hj8***,&&SY H7<D#6' - 9B3q6(KM KK  ?/   &++iizA ~~c* --/ (Md S"9&1"("2"283D3D3F"G"N"Nw"W"Hj8 8**,&& ' 9&**,&& sG J%-E D.E D0$E <D2=$E !D4">E D6!'E D8 E J%D<'D:(D<,J%.E 0E 2E 4E 6E 8E :D<< EJ%EJ% I1C%I,:I1;I4?J%IIIJ% I(%J%'I((J%,I11I44J"8J J JJ" JJ"JJ""J%cK|j|d{}|s|j|d{S|j|fi|d{S7:7!7w)z Take a screenshot of the current page. Args: page (Page): The Playwright page object kwargs: Additional keyword arguments Returns: str: The base64-encoded screenshot data N)page_need_scrolltake_screenshot_naivetake_screenshot_scroller)r-rr/ need_scrolls r0rz.AsyncPlaywrightCrawlerStrategy.take_screenshotOsa!11$77 33D99 9766tFvFF F8:Gs1AAAAA AAAAcK ddlm}||}|djd}t}|j |dt j |jjdS#t$r}dt|}|jjdd d |i tjdd d }tj |} t#j$} | j'd|d| t}|j |dt j |jjdcYd}~Sd}~wwxYww)z Convert the first page of the PDF to a screenshot. Requires pdf2image and poppler. Args: pdf_data (bytes): The PDF data Returns: str: The base64-encoded screenshot data r)convert_from_bytesrrrrz%Failed to take PDF-based screenshot: zPDF Screenshot failed: {error}rrrrrrrrrN) pdf2imagerconvertrrrrrrrr2rFrrrrrrrr) r-rrimages final_imgrrrrrrs r0take_screenshot_from_pdfz7AsyncPlaywrightCrawlerStrategy.take_screenshot_from_pdfds/ I 4'1Fq ))%0IyH NN8FN 3##H$5$5$78??H H ICCF8LM KK  8/   ))E:W=C>>#&D))+D IIh O$I OyH HHXfH -##H$5$5$78??H H Is6EA0A54E5 E>CEEEEEcK |j|d{}|d}|d}t||jdt}|j ||dd{g}|j }|d} || zdz} t | D]} | | z} | | dz k(r(|| z} | dk(rn|j || dd{|jd| d d{tjd d{|jd d d d{}tjt|jd}|j||j || dd{t!d|D}tj"d|dj$|f}d}|D]4}|j'|jdd|f||j(z }6t}|jd}|j+|dd t-j.|j1j3d}|S70777x7^7E7#t4$r}dt7|}|j8j;ddd|itj"ddd}t=j>|}tAjB}|jEd|d|t}|j+|d t-j.|j1j3dcYd}~Sd}~wwxYww)!am Attempt to set a large viewport and take a full-page screenshot. If still too large, segment the page as before. Requires pdf2image and poppler. Args: page (Page): The Playwright page object kwargs: Additional keyword arguments Returns: str: The base64-encoded screenshot data Nr$r#r3r%rrzwindow.scrollTo(0, )g{Gz?FjpegU) full_pagerqualityrc34K|]}|jywr+)r#)rDrs r0rEzJAsyncPlaywrightCrawlerStrategy.take_screenshot_scroller..s>cszz>sBMP)rrrz*Failed to take large viewport screenshot: z)Large viewport screenshot failed: {error}rrrrrrrrrrr)#rfrjrwrrirrangerrxr9rrrrrrrrr$paster#rrrrrrr2rFrrrrrr)r-rr/rrrlarge_viewport_heightsegmentsrr num_segmentsry_offsetlast_part_heightseg_shotrrstitchedoffsetrencodedrrrrs r0rz7AsyncPlaywrightCrawlerStrategy.take_screenshot_scrollers?O I#77==J#G,J$X.K %( 8:TU% !(($0EF   H ..M+H5O'?:a?L<( %. q(('2_'D$(1,00:Qa1bcccmm&9(1$EFFFmmD)))"&5vWY!ZZjj!23;;EB$/ %4((:)YZ Z Z>X>>Lyy!):):L(IJHF %s{{51Av;?#**$ % yH''.H MM(5"M =&&x'8'8':;BB7KGN}> .dF)[ [  IHQQM KK  C/   ))E:W=C>>#&D))+D IIh O$I OyH HHXfH -##H$5$5$78??H H IsMIIAII AI<I=III6I7IIAI.I/CIMI IIIIII M%CL=7M8M=MMc^K |jdd{}tj|jdS7(#t$r}dt |}|j jddd|i tjd d d }tj|}tj}|jd|d|t}|j!|dtj|j#jdcYd}~Sd}~wwxYww)z Takes a screenshot of the current page. Args: page (Page): The Playwright page instance Returns: str: Base64-encoded screenshot image F)rNrzFailed to take screenshot: zScreenshot failed: {error}rrrrrrrrrrrr)rrrrrr2rFrrrrrrrrrrr) r-rrrrrrrrs r0rz4AsyncPlaywrightCrawlerStrategy.take_screenshot_naives I#??J##J/66w? ?@ I9#a&BM KK  4/  ))E:W=C>>#&D))+D IIh O$I OyH HHXfH -##H$5$5$78??H H! IsED-AA'AD-A D* CD%D* D-%D**D-cK|jrF|jj|d{}|jjddd|i|S|jj dd y7Dw) a Exports the current storage state (cookies, localStorage, sessionStorage) to a JSON file at the specified path. Args: path (str): The path to save the storage state JSON file Returns: dict: The exported storage state )rNz Exported storage state to {path}r7rrz5No default_context available to export storage state.rirj)re storage_staterFrrm)r-rrs r0export_storage_statez3AsyncPlaywrightCrawlerStrategy.export_storage_state s{   ..<<$<GGE KK  :~   L KK  O  Hs,A5A3AA5c K |jdd{t|tr|g}n|}g}|D]u} d} |jj |d|dd{}tj} |jdd d{|j|r|nddiwd|dS77T#t $r,}dt|vr|j jdd |jd d d{7n?#t $r3}|j jd dd t|iYd}~nd}~wwxYw |jdd d{7n?#t $r3}|j jddd t|iYd}~nd}~wwxYwddd}n7|j jddd t|idt|d}Yd}~d}~wwxYw7`#t $r4}|j jddd t|iYd}~d}~wwxYw#t$rQ}|j jddd t|i|jdt|dYd}~Rd}~wwxYw#t$rA}|j jddd t|idt|dcYd}~Sd}~wwxYww)a Executes user-provided JavaScript code with proper error handling and context, supporting both synchronous and async user code, plus navigations. How it works: 1. Wait for load state 'domcontentloaded' 2. If js_code is a string, execute it directly 3. If js_code is a list, execute each element in sequence 4. Wait for load state 'networkidle' 5. Return results Args: page (Page): The Playwright page instance js_code (Union[str, List[str]]): The JavaScript code to execute Returns: Dict[str, Any]: The results of the execution r Nz (async () => { try { return await (async () => { a  })(); } catch (err) { return { success: false, error: err.toString(), stack: err.stack }; } })(); zExecution context was destroyedz6Navigation triggered by script, waiting for load stater+rrrrzNavigation wait failed: {error}rrrz!Network idle wait failed: {error}Tz6Navigation triggered, ignoring context destroyed error)r*r#Playwright execution error: {error}Fr*rrz!DOM content load timeout: {error}r*zScript chunk failed: {error}r*results Script execution failed: {error}) rrLr2rIrrrFrrmrrrr) r-rrscriptsr&scriptrrnav_errt1s r0rrz9AsyncPlaywrightCrawlerStrategy.robust_execute_user_script$s)* 7**+=> > >'3'")!G!l HkH"FEI&(,||'<'&>vu&>&U U U#(" $ 3 3,M(1,3S\+B!4!"!"" "&*&>&>$15'?'"!"!"$)" $ 3 3,O(1,3S\+B!4!"!"",0(`&F !KK--(M$-(/Q'8. 27Q%HFK%ITY  ++$G )$+SV#4,(!HKK%% >% 'Q0& NNus1v#FGGH 7 KK  :Q(   %s1v6 6  7sRKJB4!JH4#B8!B6"B8&H4;G4G1G4H4-J3K4J6B88 G.*G)-D DD  G) E)E<G)EG) E' E#!E'&G)' F#0)FG)F##AG)#H4)G..H41G44 H1=)H,&H4,H11H44 J=AJ J JJ K6KKKKKc K |jdd{t|tr|g}n|}g}|D]} |jj |d|dd{}t j }|jddd{t j }|jddd{|j |r|ndd id |dS777U7(#t$rQ}|jjd d d t|i |j dt|dYd}~d}~wwxYw#t$rA}|jjdd d t|i dt|dcYd}~Sd}~wt$rA}|jjdd d t|i dt|dcYd}~Sd}~wwxYww)a; Executes user-provided JavaScript code with proper error handling and context. Args: page: Playwright page object js_code: Single JavaScript string or list of JavaScript code strings Returns: Dict containing execution status and results/errors r Nz (() => { return new Promise((resolve) => { try { const result = (function() { a' })(); // If result is a promise, wait for it if (result instanceof Promise) { result.then(() => { // Wait a bit for any triggered effects setTimeout(() => resolve({ success: true }), 100); }).catch(error => { resolve({ success: false, error: error.toString(), stack: error.stack }); }); } else { // For non-promise results, still wait a bit for effects setTimeout(() => resolve({ success: true }), 100); } } catch (error) { resolve({ success: false, error: error.toString(), stack: error.stack }); } }); })() rrrr*Tr#r+rrFr$r%r') rrLr2rIrrrrrFrr) r-rrr(r&r)rr+rs r0execute_user_scriptz2AsyncPlaywrightCrawlerStrategy.execute_user_scriptsV 7**+=> > >'3'")!G!9 H8H#'<<#8#8) *01 !#$#FLB223Et2TTTB22=$2OOONNV6)T9JKa9 Hv $8 8I ?#NUPHKK%% E% 'Q0& NNus1v#FGGH 7 KK  :Q(   %s1v6 6 7 KK  :Q(   %s1v6 6  7sGEC!E#C'C!.C' C#.C'<C%=C'EGE!C'#C'%C'' E0AD<6E<EE G 6F GG G6G G GGGcVK|jj|dd{S7w)z Checks if an element is visible on the page. Args: page: Playwright page object Returns: Boolean indicating visibility a () => { const element = document.body; if (!element) return false; const style = window.getComputedStyle(element); const isVisible = style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'; return isVisible; } NrIrr-rs r0r`z/AsyncPlaywrightCrawlerStrategy.check_visibilitys0\\**4     )')cK|j|||d{}|dr|j|dzd{|S7'7w)z Safely scroll the page with rendering time. Args: page: Playwright page object x: Horizontal scroll position y: Vertical scroll position Nr*r") csp_scroll_tory)r-rxyr8rs r0rz*AsyncPlaywrightCrawlerStrategy.safe_scroll7sM))$155 ) '' 5 5 5 6 5sAA AAAAcK |jj|d|d|d|d|d|d|d d{}|d s/|jjd d d |j d i |S7:#t $rA}|jj dd d t|i dt|dcYd}~Sd}~wwxYww)a9 Performs a CSP-compliant scroll operation and returns the result status. Args: page: Playwright page object x: Horizontal scroll position y: Vertical scroll position Returns: Dict containing scroll status and position information z() => { try { const startX = window.scrollX; const startY = window.scrollY; window.scrollTo(z, a); // Get final position after scroll const endX = window.scrollX; const endY = window.scrollY; return { success: true, startPosition: { x: startX, y: startY }, endPosition: { x: endX, y: endY }, targetPosition: { x: z, y: z\ }, delta: { x: Math.abs(endX - z6), y: Math.abs(endY - a) } }; } catch (e) { return { success: false, error: e.toString() }; } }Nr*z Scroll operation failed: {error}SCROLLrrz!Failed to execute scroll: {error}Fr$)rIrrFrmrwrrr2)r-rr4r5rrs r0r3z,AsyncPlaywrightCrawlerStrategy.csp_scroll_toEs. 7<<00)*+2aS 1345#U1#>445374453 7#F<)$ ##> #VZZ%89$ MKN 7 KK  ;Q(   %s1v6 6  7sDC2A3A19A30C1A33 B=<6B82B=3C8B==CcVK|jj|dd{S7w)z Get the dimensions of the page. Args: page: Playwright page object Returns: Dict containing width and height of the page z () => { const {scrollWidth, scrollHeight} = document.documentElement; return {width: scrollWidth, height: scrollHeight}; } Nr/r0s r0rfz2AsyncPlaywrightCrawlerStrategy.get_page_dimensionss0\\**4     r1c K |jj|dd{}|S7#t$r3}|jj dddt |iYd}~yd}~wwxYww)z Determine whether the page need to scroll Args: page: Playwright page object Returns: bool: True if page needs scrolling z () => { const scrollHeight = document.documentElement.scrollHeight; const viewportHeight = window.innerHeight; return scrollHeight > viewportHeight; } NzDFailed to check scroll need: {error}. Defaulting to True for safety.r7rrT)rIrrrFrmr2)r-rrrs r0rz/AsyncPlaywrightCrawlerStrategy.page_need_scrollsw  $ 5 5d!K   KK  ^Q(     s6A+,*,A+, A()A#A+#A((A+)NNN)rErrFrrNr!)ror2)rsr2rtr)rsr2)r|r2)rzDict[str, str])r)rrrr2rrG)rrrr2rrGr3r2r.r2rrr3r)r!N)rrrnrGroz Optional[int])rrrz'VirtualScrollConfig')rrr3None)rrr3bytes)rrr3z Optional[str])rrrr2r3z"List[Dict[str, Union[str, float]]]r+)rr2rrr3tuple)rr=r3r2)rrr3r2)rr2r3r)rrrzUnion[str, List[str]]r3Dict[str, Any])r!)rrr4rhr5rhr8rG)rrr4rhr5rhr3r?)rr)rrr3bool)(r4r5r6r7rOrTr^rRrYrnrurcr}rrrrrr1rrmrqrrrrrrrr rrr!rrr-r`rr3rfrr,r8r0r:r:-s:sw* +* BEJ,,.1,@0II 0I IVO #O # 0O # O #dL:\hV/b!F >@" " %(" +" J59EE!1E ENG*"IH]IBID2T7T7#8T7 T7lc7c7#8c7 c7J 0 :7x &r8r:ceZdZdZy)HTTPCrawlerErrorz5Base error class for HTTP crawler specific exceptionsNr4r5r6r7r,r8r0rBrBs?r8rBceZdZdZy)ConnectionTimeoutErrorz%Raised when connection timeout occursNrCr,r8r0rErEs/r8rEc$eZdZdZdfd ZxZS)HTTPStatusErrorz"Raised for unexpected status codesc>||_t| d|d|y)NzHTTP r)rsuperrO)r-rrk __class__s r0rOzHTTPStatusError.__init__s%& 5 Ry9:r8)rrhrkr2)r4r5r6r7rO __classcell__)rJs@r0rGrGs,;;r8rGc eZdZUdZdZdZded<dZded<ede jxsd d zZ ded <d Z ded <e hdZded<edddddddZded<dde e ef d(dZd)dZd*dZej*dZd+dZ d,dZd*d Zd*d!Zd-d"Zd.d#Zd/d$Zd0d%Z d1d&Z d2 d3d'Z y)4AsyncHTTPCrawlerStrategyzW Fast, lightweight HTTP-only crawler strategy optimized for memory efficiency. )rFmax_connections dns_cache_ttl chunk_size_sessionrKrEz Final[int]DEFAULT_TIMEOUTiDEFAULT_CHUNK_SIZErrrDEFAULT_MAX_CONNECTIONS,DEFAULT_DNS_CACHE_TTL>rawfilehttphttpsr VALID_SCHEMESz?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zen-US,en;q=0.5zgzip, deflate, brz keep-alive1z.r8rccyr+r,rfs r0rz3AsyncHTTPCrawlerStrategy.__init__..rgr8rdcyr+r,rfs r0rz3AsyncHTTPCrawlerStrategy.__init__..rgr8) rrErFrNrOrPrQr' _execute_hookrKru)r-rErFrNrOrPks r0rOz!AsyncHTTPCrawlerStrategy.__init__s-C0A0C .*$9= E  wt))1- -  &(DE o'CD j">? sBcBK|jd{|S7wr+rQrSs r0rTz#AsyncHTTPCrawlerStrategy.__aenter__rUrVc@K|jd{y7wr+rXrZs r0r^z"AsyncHTTPCrawlerStrategy.__aexit__r_r`cK |js|jd{|jy7#wxYwwr+)rQrRrSs r0_session_contextz)AsyncHTTPCrawlerStrategy._session_contexts9 ==jjl""--  # s ?:8:?:<?c||jvr%t|j|||j|<ytd|)Nrq)rKr'rjrr)r-rs hook_funcs r0ruz!AsyncHTTPCrawlerStrategy.set_hook s>  "$+D,>,> 9$UDJJy !29+>? ?r8chKtj|r||i|d{S||i|S7 wr+)rxry)r-rsrqrzr/s r0rjz&AsyncHTTPCrawlerStrategy._execute_hook s>  & &y 1"D3F33 3$)&))4s !20 2cK|jsrtj|j|jdd}tj t |j|t|j|_yyw)NTF)limit ttl_dns_cache use_dns_cache force_close)total)r connectorr) rQaiohttp TCPConnectorrNrO ClientSessionrr`r$rS)r-rys r0rRzAsyncHTTPCrawlerStrategy.start sm}},,**"00"! I $11T//0#%D,@,@ADMsBBcrK|jrW|jjs@ tj|jj dd{d|_yyy7#tj $r,|j r|j jddYJwxYw#d|_wxYww)NrFrzSession cleanup timed outCLEANUPrj)rQclosedrxrrYrrFrmrSs r0rYzAsyncHTTPCrawlerStrategy.close' s ==!5!5 %&&t}}':':'Ktjj|std|g}|j |23d{}|j |j jdd:756tdj|idSw)Nrrrrrrrr) rrrrrrtobytesrrr~)r-rrrs r0 _handle_filez%AsyncHTTPCrawlerStrategy._handle_file9 sww~~d##&$r:} |j d| d{7tAdt9| d} ~ wt<jB$r:} |j d| d{7tEdt9| d} ~ wt<jF$r:} |j d| d{7tIdt9| d} ~ wtJjLjN$r:} |j d| d{7tAdt9| d} ~ wtP$r:} |j d| d{7tIdt9| d} ~ wwxYwdddd{7y#1d{7swYyxYww)NrrR)rxconnect sock_read)rallow_redirectssslrproxyPOSTrjsonrbrrVzUnexpected status code for rrrr)rrrr?rcrdzRequest timed out: zConnection failed: zHTTP client error: zHTTP request failed: ))ror$r]rSrr`rErupdatefollow_redirects verify_sslrrrrrrKrrrrrGcharsetchardetdetectrrrr2r.rzServerTimeoutErrorrEClientConnectorErrorConnectionError ClientErrorrBrx exceptionsrr) r-r.rsessionrrrequest_kwargs proxy_urlrr:rrrs r0 _handle_httpz%AsyncHTTPCrawlerStrategy._handle_httpb s1 ((*J IJ Ig#))AT-A-AG 4--.G""**t22::;##'#6#6#G#G**55" NI"" 2263F3FG *3w'""))V3&&++-1-@-@-E-EN6*&&++-1-@-@-E-EN6*.$**-.sNC C C* I*7??4+>+>+E+Es]n]""ai(x}})>?G8??8S8-$OO9#? (//H##*>>'//2C#DZ#P#[T[/$__.55hy5Q)-h.>.>)?$,OO'*8<<'8 F6$**_5f===!+"""CJ IJ IJ I< D")>&>)"CJ IB"""".-- M,djj,Q///,/B3q6(-KLL// F,djj,Q///%(;CF8&DEE&& G,djj,Q///&)s1vh'GHH IQJ IJ IJ IJ IJ IsfR*KR*E$R?K!R,L1K$2L5K0K' C&K05K*6K0; LK,L R*K.R*!R$L'K0*K0,L.R*0L 6K97L >LRRM2L53MR%N=N>NR0O%O  O%% RP:P P:: RQ;Q!Q;;RR R*RR*R'R R'#R*cTK|xstj|}t|}|jj d}||j vrt d| |dk(r#|j|jd{S|dk(r4|jdr|ddn|dd}|j|d{S|j||d{S7W7 7#t$r<}|jr*|jjdd t||d  d}~wwxYww) N/zUnsupported URL scheme: rYrXrrrzCrawl failed: {error}CRAWL)rr.r)rrHr%schemerstripr\rrrrrrrrrFrr2)r-r.rr/parsedr raw_contentrs r0r1zAsyncHTTPCrawlerStrategy.crawl s% ?+77?#%%c* ++ +7x@A A !..v{{;;;5*-)Ac!"gs12w !--k:::!..sF;;;< ;; {{ !!3%(VC8"  sxAD(#C CC D(4C ;C<C ?D(C CC D(C C C D%)7D  D%%D() rEzOptional[HTTPCrawlerConfig]rFzOptional[AsyncLogger]rNrhrOrhrPrh)r3rM)r3r<)rsr2rqrr3r<) rsr2rqrrzrr/rr3r)rr2r3z AsyncGenerator[memoryview, None])rr2r3r)r:r2r3rr:r;r+)r.r2rzOptional[CrawlerRunConfig]r3r)!r4r5r6r7 __slots__rS__annotations__rTrjr cpu_countrUrW frozensetr\r&r`rOrTr^ contextlibasynccontextmanagerrorurjrRrYrrrrrr1r,r8r0rMrMstI"$OZ$%. .*-b<2<<>3FQ!2K*LZL(+:+$%EFM5F+S+."%(T -M57;(,62, @3@&@ @  @  @6##$@ * * * *  * *  %(   &OI OI!OI  OIh.2   +   r8rM)H __future__rrxrrabcrrtypingrrrr r r r r rplaywright.async_apirrrrziorPILrrrrWr js_snippetrmodelsrrrrrrr async_loggerrr=ruser_agent_generatorrrMr rNr!r"r#rrzraiohttp.clientr$ urllib.parser%typesr&r functoolsr'r)r:rrBrErGrMr,r8r0rs" #3322 ,G++ &&.MM%+2+QQ(!" 3 B"%9B"RD y  - ;&;L3Lr8