/i"ddlmZmZmZmZmZmZmZddlZddl Z ddl m Z ddl Z ddl mZmZddlmZddlmZmZddlmZGd d eZGd d eZGd deZGddZdZedk(re j<eyy))ListOptionalUnionAsyncGeneratorDictAnyCallableN)urljoin) BrowserConfigCrawlerRunConfig) CrawlResult) AsyncLoggerLogLevel)hooks_to_stringceZdZdZy)Crawl4aiClientErrorz1Base exception for Crawl4ai Docker client errors.N__name__ __module__ __qualname____doc__R/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/crawl4ai/docker_client.pyrr ;rrceZdZdZy)ConnectionErrorz2Raised when connection to the Docker server fails.Nrrrrrrs<rrceZdZdZy) RequestErrorz1Raised when the server returns an error response.Nrrrrr r rrr ceZdZdZ ddededededeef dZd ed dfd Z dd Z d d e edee dee deeeeefeeeffded eeeff dZdeded ej*fdZ d d e edee dee deeeeefeeeffded eee eeedfff dZd eeeffdZddZd!dZdeedeedeed dfdZy)"Crawl4aiDockerClientzMClient for interacting with Crawl4AI Docker server with token authentication.Nbase_urltimeout verify_sslverboselog_filec|jd|_||_t|tj ||_tj||ddi|_ d|_ y)N/)r' log_levelr&z Content-Typeapplication/json)r$verifyheaders) rstripr#r$rrDEBUGloggerhttpx AsyncClient _http_client_token)selfr#r$r%r&r's r__init__zCrawl4aiDockerClient.__init__s[!,  !8x~~W^_ !--#%78  &* remailreturnchKt|jd} |jjd|d|jj |d|id{}|j |j}|d|_d |j|jjd <|jjd dy7r#tjtjf$r;}d t|}|jj|d t!|d}~wwxYww)z1Authenticate with the server and store the token.z/tokenzAuthenticating with email: AUTHtagr7jsonN access_tokenBearer AuthorizationzAuthentication successfulzAuthentication failed: ERROR)r r#r0infor3postraise_for_statusr>r4r-successr1r HTTPStatusErrorstrerrorr)r5r7urlresponsedatae error_msgs r authenticatez!Crawl4aiDockerClient.authenticate1sdmmX. - KK  :5'B  O!..33Cw>N3OOH  % % '==?D~.DK;B4;;-9PD   % %o 6 KK   ;  H P ""E$9$9: -1#a&:I KK  iW  5!), , -s<D2ACCA1CD2C#D/46D**D//D2cK |jjt|jdd{|jj d|jdy7/#t j$rE}|jjdt|dtdt|d}~wwxYww) z6Check if server is reachable, raising an error if not.z/healthNz Connected to READYr;zServer unreachable: rBzCannot connect to server: ) r3getr r#r0rFr1r rIrHr)r5rMs r _check_serverz"Crawl4aiDockerClient._check_serverAs I##'' y(IJ J J KK  -  ?W  M K!! I KK   4SVH=7  K!$>s1vh"GH H Is9C2A(A&.A(%C&A((C;AB;;CCurlsbrowser_configcrawler_confighooks hooks_timeoutc.|jr&d|j|jjd<||r|jni|r|jnid}|r6t d|j Dr t |}n|}||d|d<|S)z"Prepare request data from configs.r@rA)rTrUrVc32K|]}t|ywN)callable).0vs r z8Crawl4aiDockerClient._prepare_request.._s718A;7s)coder$rW)r4r3r-dumpanyvaluesr)r5rTrUrVrWrX request_data hooks_codes r_prepare_requestz%Crawl4aiDockerClient._prepare_requestJs ;;;B4;;-9PD   % %o 67En11327En1132  7 77,U3 # #(%L ! rmethodendpointcKt|j|} |jj||fi|d{}|j |S7#t j $r}tdt|d}~wt j$r}tdt|d}~wt j$r}d|jjjddvr3|jjjdt|n t|}td|jjd |d}~wwxYww) z)Make an HTTP request with error handling.NzRequest timed out: zFailed to connect: r+z content-typedetailz Server error z: )r r#r3requestrEr1TimeoutExceptionrrHr rGrKr-rRr> status_code)r5rgrhkwargsrJrKrMrNs r_requestzCrawl4aiDockerClient._requestms.dmmX. V6T..66vsMfMMH  % % 'ON%% B!$7Ax"@A A!! B!$7Ax"@A A$$ V-1C1C1G1GXZ1[[*..xQ@ V qzz/E/E.Fb TU U  VsQE!AAAEAE'A>>EB++EB E  EEc Kjd{j||||| |xr |j}jj dt |d|rdndd|rdt tdff fd }|Sjd d | d{}|j} | jd dstd| jdd| jdgD cgc] } tdi| } } jjdt | ddt | dk(r| dS| S7=7cc} ww)a Execute a crawl operation. Args: urls: List of URLs to crawl browser_config: Browser configuration crawler_config: Crawler configuration hooks: Optional hooks - can be either: - Dict[str, Callable]: Function objects that will be converted to strings - Dict[str, str]: Already stringified hook code hooks_timeout: Timeout in seconds for each hook execution (1-120) Returns: Single CrawlResult, list of results, or async generator for streaming Example with function hooks: >>> async def my_hook(page, context, **kwargs): ... await page.set_viewport_size({"width": 1920, "height": 1080}) ... return page >>> >>> result = await client.crawl( ... ["https://example.com"], ... hooks={"on_page_context_created": my_hook} ... ) Nz Crawling z URLs z (streaming)rjCRAWLr;r8c Kjjdjd4d{}|j|j 23d{}|j st j|}d|vr1jj|jdd|ddjj|jddd|jd d  |jd d k(rtdi|776dddd{7y#1d{7swYyxYww)NPOSTz /crawl/streamr=rIrJunknown)rJrITtimingg)rJrFrvstatus completedr) r3streamr#rE aiter_linesstripr>loadsr0 error_statusrR url_statusr)rKlineresultrLr5s rstream_resultsz2Crawl4aiDockerClient.crawl..stream_resultss4,,33Ft}}o]<[bf3g <  <EE rtz/crawl)r>r$rFFzCrawl failed: msgz Unknown errorresultszCrawl completed with z resultsr rr) rSrfryr0rClenrrrpr>rRr rF) r5rTrUrVrWrX is_streamingrrK result_datarrrLs ` @rcrawlzCrawl4aiDockerClient.crawl~s`B  """$$T>>5R_`%?.*?*?  9SYKv|mY[5\]cjk  <.d9J*K <"# #vxdMZZmmo y%0 0W/XYZ Z-8__Y-KL;##LL 3CL>JPWX \Q.wqz;G;? #0[ Ms6EEBE)E*AEEAEEEcbK|jddd{}|jS7w)zRetrieve configuration schemas.GETz/schemaN)rpr>)r5rKs r get_schemazCrawl4aiDockerClient.get_schemas*ui88}}9s /-/cK|jjdd|jjd{y7w)zClose the HTTP client session.zClosing clientCLOSEr;N)r0rCr3acloser5s rclosezCrawl4aiDockerClient.closes6 )w7&&(((s;AAAcK|Swr[rrs r __aenter__zCrawl4aiDockerClient.__aenter__s  sexc_typeexc_valexc_tbc@K|jd{y7wr[)r)r5rrrs r __aexit__zCrawl4aiDockerClient.__aexit__sjjls )zhttp://localhost:8000g>@TTN)r8N)NNN)r8r") rrrrrHfloatboolrr6rOrSrr r rrr intrrfr1Responserprrrrrrtype Exceptionrrrrr"r"sW0"& *** *  * 3- *$--- I3759FJ !3i!!/!!!12 ! d3=14S>ABC !  ! c3h!FVSVCVennV(3759FJ @<3i@<!/@<!!12 @< d3=14S>ABC @<  @< {D-~k4>O/PP Q@ urllib.parser asyncio async_configsr r modelsr async_loggerrrutilsrrrrr r"rrrunrrrrs}MMM :/" )  )  & rrl zGKKr