/i+dZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl m Z mZmZddl mZddlmZmZmZmZmZmZmZddlmZmZddlZddlZ ddlmZ dd lm!Z!d Z" ddl$Z$d Z% ddl&Z&d Z'd d l(m)Z)m*Z*ddlm+Z+e+rd dl,m-Z-dZ.edZ/e j`de jbZ2e j`de jbZ3e j`de jbe jhzZ5e j`de jbZ6ddZ7 d d!dZ8d"dZ9 d#dZ:d$dZ;d%dZ<GddZ=y#e#$rd Z"YwxYw#e#$rd Z%YwxYw#e#$rd Z'YwxYw)&u async_url_seeder.py Fast async URL discovery for Crawl4AI Features -------- * Common-Crawl streaming via httpx.AsyncClient (HTTP/2, keep-alive) * robots.txt → sitemap chain (.gz + nested indexes) via async httpx * Per-domain CDX result cache on disk (~/.crawl4ai/__.jsonl) * Optional HEAD-only liveness check * Optional partial download + meta parsing * Global hits-per-second rate-limit via asyncio.Semaphore * Concurrency in the thousands — fine on a single event-loop ) annotationsN)datetime timedeltatimezone)Path)AnyDictIterableListOptionalSequenceUnion)quoteurljoin)html)etreeTF)AsyncLoggerBase AsyncLogger) TYPE_CHECKING SeedingConfigz+https://index.commoncrawl.org/collinfo.json)dayszu]*?(?:name|property|http-equiv)\s*=\s*["\']?([^"\' >]+)[^>]*?content\s*=\s*["\']?([^"\' >]+)[^>]*?)\/?>z&]*charset=["\']?([^"\' >]+)z(.*?)z=]*rel=["\']?([^"\' >]+)[^>]*href=["\']?([^"\' >]+)c tr3tj|}|jd}|r t |Sy#t $rYywxYw)z1Extract the most recent lastmod from sitemap XML.z"//*[local-name()='lastmod']/text()N)LXMLr fromstringxpathmax Exception) xml_contentrootlastmodss U/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/crawl4ai/async_url_seeder.py_parse_sitemap_lastmodr%QsR  ##K0Dzz"FGH8}$     s8< AAcJ|jsy t|d5}tj|}dddj ddk7ry|dkDret j |djdd }t jtj|z jd z }||kDry|r|r|j d }|r||kDry|j d ddk(ryy #1swYxYw#tjtttf$rYywxYw)z Check if sitemap cache is still valid. Returns False (invalid) if: - File doesn't exist - File is corrupted/unreadable - TTL expired (if ttl_hours > 0) - Sitemap lastmod is newer than cache (if validate_lastmod=True) FrNversionrr created_atZz+00:00isitemap_lastmod url_countT)existsopenjsonloadgetr fromisoformatreplacenowrutc total_secondsJSONDecodeErrorKeyError ValueErrorIOError) cache_path ttl_hoursvalidate_lastmodcurrent_lastmodfdatar) age_hourscached_lastmods r$_is_cache_validrC`s     *c " a99QA AA AAcdtjtjj ||t ||d} t |d5}tj||dddy#1swYyxYw#t$rYywxYw)z"Write URLs to cache with metadata.r)r(r)r+ sitemap_urlr,rEwN) rr4rr5 isoformatlenr.r/dumpr )r;rErHr+r@r?s r$ _write_cacherMs|ll8<<0::<*"Y  D *c " a IIdA        s0 A;A/&A;/A84A;8A;; BBctj||ry|jddd}tj||xs,|jdxrtj|dd|S)NT://rwww.)fnmatchsplit startswith)urlpatterncanons r$_matchrYsfsG$ IIeQ  #E OOE7 + R  (PW__U12Y-PScb tr t|tr|jdd}t j |}|jd-|jdjxsdjnddiigd}|jdD]}|jjdxs8|jjdxs|jjd }|r1|jjd d|d |j<d |jvs|jd j|d <|jd D]}|jjdd}|s"|jj!}dDcic] }||jvr||j|"}}|D]&} |dj#| gj%|(|jdD]Q} | js t'j(| jj} |dj%| S|jd} | | jjdd|d<|Sddiigdd}t,j/|} | r| j1djnd|d<t2j5|D]\}}||d |j<t6j/|} | r| j1djnd|d <t8j5|D]9\} }|dj#| jgj%d|i;t;j<dt:j>t:j@z}|j5|D]:} t'j(|j} |dj%| <t;j.d|t:j>}|r|j1d|d<|S#t tjf$ricYSwxYwcc}w#t&j*$rYwxYw#t&j*$rYwxYw)Nutf-8r3z.//title)titlecharsetmetalinkjsonldz.//metanamepropertyz http-equivcontentr`r_z.//linkrel)hrefastypehreflangraz&.//script[@type="application/ld+json"]rbz.//htmllang)r^r_r`rarbrkrr^rgzC]*type=["\']application/ld\+json["\'][^>]*>(.*?)z ]*lang=["\']?([^"\' >]+))!r isinstancestrencode lxml_htmlrr9r ParserErrorfindtextstriprattribr1lowerrT setdefaultappendr/loadsr7 _title_rxsearchgroup_meta_rxfindall _charset_rx_link_rxrecompileIS)srcdocinfoelkrel_attr rel_valuesaentryrfscript jsonld_data html_elemmvrgjsonld_patternmatch lang_matchs r$ _parse_headrsI  #s#jj)4&&s+C xx #/hhz*//52<<>59b   ))I& ?B f%;*;!yy}}\: *,))-- 2*FV QWWY'bii'"$))I"6"<"<">Y  ?))I& ?Byy}}UB/H!)//1J/2E56"))^ ! _EEE" ?V ''R077> ? ?ii HI F{{"&**V[[->->-@"AKN))+6  HHY'  $++//;DL %)d$&b"NDA*+AGGAJ$$&DM  %$1"#V QWWY$3A,-aggaj&&(4DO%%c*H T V  R077GHZZNPRPTPTWYW[W[P[]N'', **U[[]3K N ! !+ .>RTTJJ!''*V KE--. I *E++2##   s<7Q(%Q9AQ>"7RQ65Q6>RRR.-R.cbeZdZdZeddddf ddZdddZddZd dZd!dZ d"dZ d#d Z d$ d%d Z d&d Z d'd Zd(d Zd)d(dZd*dZd+dZ d, d-dZd.dZ d/ d0dZd1dZd2dZd3dZd4dZdZdZdZd5dZy)6AsyncUrlSeedera Async version of UrlSeeder. Call pattern is await/async for / async with. Public coroutines ----------------- await seed.urls(...) returns List[Dict[str,Any]] (url, status, head_data) await seed.many_urls(...) returns Dict[str, List[Dict[str,Any]]] await seed.close() closes the HTTP client if owned by seeder Usage examples -------------- # Manual cleanup: seeder = AsyncUrlSeeder() try: urls = await seeder.urls("example.com", config) finally: await seeder.close() # Using async context manager (recommended): async with AsyncUrlSeeder() as seeder: urls = await seeder.urls("example.com", config) # Reusing existing client: client = httpx.AsyncClient() seeder = AsyncUrlSeeder(client=client) urls = await seeder.urls("example.com", config) # No need to close seeder, as it doesn't own the client Nc||_|du|_|xstjddddi|_||_t j|xs(tjdtj|_ |jdz dz |_ |jjdd |jd z |_d|_d|_ttj"j%|xsd |_|j&d z jdd |j&d z jdy)NTz User-AgentzpMozilla/5.0 (Windows NT 10.0; Win64; x64) +AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36)http2timeoutheadersCRAWL4_AI_BASE_DIRECTORYz .crawl4ai seeder_cache)parentsexist_okzlatest_cc_index.txtz~/.cache/url_seederlivehead)r)ttl _owns_clienthttpx AsyncClientclientloggerpathlibrosgetenvhomebase_directory cache_dirmkdirindex_cache_pathindex_id _rate_sempath expanduser cache_root)selfrrrrrs r$__init__zAsyncUrlSeeder.__init__'s-"dN  1 1b MS !   %ll>,5RYY & >56,,{:  TD9 $ !!"(, 6:rww11  // 12 6 !(((E 6 !(($(7rZc |jr6t|j|d}|r||||jdiyyy)z?Helper to log messages using the provided logger, if available.Nparams)messagetagr)rgetattrr1)rlevelrrkwargs log_methods r$_logzAsyncUrlSeeder._logHsB ;; eT:J7"(**Xr":< rZctj|jj}|j|z |dz S)N.json)hashlibsha1rn hexdigestr)rkindrVhs r$ _cache_pathzAsyncUrlSeeder._cache_pathSs9 LL & 0 0 2%1#U 33rZcK|j||}|jsytj|jjz |j j kDry tj|d4d{}tj|jd{cdddd{S7@77 #1d{7swYyxYw#t$rYywxYww)Nr') rr-timestatst_mtimerr6aiofilesr.r/rxreadr )rrrVpr?s r$ _cache_getzAsyncUrlSeeder._cache_getWs   T3 'xxz 99;qvvx(( (488+A+A+C C }}Q, 2 2zz.1 2 2 2"0 2 2 2 2  sA/C72C( C  C(#C3C 4C; C(CC( C7 C(CC(C%C C%!C($C7%C(( C41C73C44C7c<K tj|j||d4d{}|jt j |dd{dddd{y7C77 #1d{7swYyxYw#t $rYywxYww)NrI),:) separators)rr.rwriter/dumpsr )rrrVr@r?s r$ _cache_setzAsyncUrlSeeder._cache_setcs }}T%5%5dC%@#F G G!ggdjj*EFFF G G GF G G G G   sB*B A2B *A8A4A8! B ,A6-B 1B2B 4A86B 8B >B ?B B B B BBBBc !K|jxsd|j}|j|j|j}d|j }|j _|j |j |jn#jrjjnd!|j |jnd|j|j|jt|dd_t|dd _jr=t!jd r'|j|jj_j"j%d{_|j'd d d h}D]&}||vst)d|ddj+||r<|dkrj-dddd_n"t1j2|_nd_j-dd|ddfdt5dt7d|dz}t1j8|t1j:t1j: t=|j> fd} d, !f d } g} t1j@| } tC|D cgc]} t1j@| | }} t1jD| g|d{j+d{j-dd!tG| d"drrd#k(rjI| |d{} btG| }| Dcgc]}|jKd$dk\s|} }|tG| kDr$j-dd%|tG| z d&d| jMd'd (j-dd)tG| d*dnrsj-dd+ddkDr| dS| S7cc} w7'77cc}ww)-a. Fetch URLs for a domain using configuration from SeedingConfig. Parameters ---------- domain : str The domain to fetch URLs for (e.g., "example.com") config : SeedingConfig Configuration object containing all seeding parameters *NFrPcache_ttl_hoursvalidate_sitemap_lastmodTverbose+ccsitemapzInvalid source 'z'. Valid sources are: z, rwarning7hits_per_sec must be positive. Disabling rate limiting.URL_SEEDrrz6Starting URL seeding for {domain} with source={source})domainsourcerrcKdvr6jdddj23d{}|dvr7jdddj23d{}|yy7F6@76 w)NrdebugzFetching from sitemaps...rrrzFetching from Common Crawl...)r_from_sitemaps_from_cc)urforcerWrsourcess r$genz AsyncUrlSeeder.urls..gensG# '#>J O#2267EJ!Gw '#B(*#}}VWeD!G J DsI,BA<A:A<2B)B-A>.B1 B:A<<B>BB'dmaxsizec K 23d{}|vrjddd|id%jrjdddn+j|j|d{vj jdd dy77,6+#t$r*}jd d d t |idYd}~Yd}~wwxYw#j jdd dwxYww) NrSkipping duplicate URL: {url}rVrrrz(Producer stopping due to max_urls limit.rerrorz&Producer encountered an error: {error}zProducer finished.)ris_setaddputr rmset)rer producer_donequeueseenr stop_events r$producerz%AsyncUrlSeeder.urls..producers  I"u ' '!Dy '+J*/"E !((* "$NT^"`HHQK))A,&& !!# '#7Z H! '' % < '#K!3q6U+0:<< <!!# '#7Z HsmD B-B+B'B+A(B-<B)=B-%D 'B+)B-+B-- C 6 CC#C  C##&D  D c  K jrjry tj j dd{}dkDrt|k\rw jddd j j js2 j j js2y jrH j4d{ j||  d{dddd{n" j||  d{ j`7#tj $rYzt $r+} jdddt|idYd}~d}~wwxYw#tj$rYywxYw777#1d{7swYxYw7w) Nrrz,Worker failed to get URL from queue: {error}rrrrz&Worker stopping due to max_urls limit.r)emptyrasynciowait_forr1 TimeoutErrorr rrmrKr task_done get_nowait QueueEmptyr _validate)res_listrVr extract_headfilter_nonsense head_timeout live_checkmax_urlsrqueryrscore_thresholdscoring_methodrrrs r$workerz#AsyncUrlSeeder.urls..workers;;=]%9%9%; ' 0 0a @@Ca>#~~>>"nnS(J -97E?\j-<>>>>>> ..h L)5wXf)8:::!Y A++ IIg'U%s1v_/4>@, '11"!" >>>>>> :s#G*'E(E%E(AG*+ F4 G*G*9G :G*=GGG G*+G,"G*G(G*%E((F1;G*>F1 F,&G*,F11G*4G G* G  G*GG*G%G G%!G*z6Finished URL seeding for {domain}. Total URLs: {count})rcountbm25relevance_scorez:Filtered {filtered} URLs below score threshold {threshold})filtered thresholdc&|jddS)Nrr1xs r$z%AsyncUrlSeeder.urls..squu->'DrZkeyreversez;Sorted {count} URLs by relevance score for query: '{query}')rrzTQuery provided but extract_head is False. Enable extract_head for relevance scoring.r List[Dict[str, Any]])'rWrrr  concurrency hits_per_secrrrrrrrr_cache_ttl_hours_validate_sitemap_lastmodhasattrr _latest_indexrTr9joinrrr SemaphoreminrQueueEventrfilter_nonsense_urls create_taskrangegatherrK_apply_bm25_scoringr1sort)"rrconfigrr&r' valid_sourcess queue_sizerrresults prod_task_workersoriginal_countr'r rrrrrrrWrrrrrrrrrs"`` @@@@@@@@@@@@@@@@@r$rEzAsyncUrlSeeder.urlsls7..'C&& ** ((  ** \\  $*NN$>&..#';;DKK  E &,oo&A6??r  00..!(0A2 F)09SUY)Z& ;;74;; :v~~?Y"(..DKK  == "&"4"4"66DM,,s#y)  \A % &qc)? -@X?YZ\\ \ q  X^hj!%!(!2!2 44WfEEG*!$W&-`7H!1LP_1_1``!CL0IIf&b2@3w<2O^m%nt~@ LLDdL S IIf['*7|eD*  V < IIqxB  C&.\wy!>w>a7b0 2F asuE2QQ'Q0D9Q)"Q Q&Q 'Q>Q?AQQQQ3Q7BQQQQQcK|jdddt|id|jr=t|jdr'|j|j|j_|Dcgc]}|j ||}}t j|d{}tt||}|jddd |Scc}w74w) a> Fetch URLs for many domains in parallel. Parameters ---------- domains : Sequence[str] List of domains to fetch URLs for config : SeedingConfig Configuration object containing all seeding parameters Returns a {domain: urls-list} dict. rz+Starting URL seeding for {count} domains...rrrrNz*Finished URL seeding for multiple domains.r) rrKrr*rrErr4dictzip)rdomainsr7rtasksr; final_resultss r$ many_urlszAsyncUrlSeeder.many_urls's" &G!3w<0j  B ;;74;; :v~~?Y"(..DKK "  IIff %   ..S'23  @j  R /sA.C0C C C!5Cc Kddlm}|d|d|_d_j ddd t id j rPj d krj d dd d_n,tjj _nd_tdtd|dz}tj|tjtjtg}fd}d fd } tj|} g} t!|D].} tj| |} | j#| 0| d{j%d{| D]}|j'tj(| ddid{j*r)j,dk(rj/|d{}j0/|Dcgc]$}|j3dd j0k\s#|&}}t5d|Dr|j7ddj ddt t |Dcgc]}|j3ddk(s|c}dd |S767!77cc}wcc}ww)!au Extract head content for a custom list of URLs using URLSeeder's parallel processing. This method reuses URLSeeder's efficient parallel processing, caching, and head extraction logic to process a custom list of URLs rather than discovering URLs from sources. Parameters ---------- urls : List[str] List of URLs to extract head content from config : SeedingConfig, optional Configuration object. If None, uses default settings for head extraction concurrency : int, default=10 Number of concurrent requests timeout : int, default=5 Timeout for each request in seconds Returns ------- List[Dict[str, Any]] List of dictionaries containing url, status, head_data, and optional relevance_score NrrTF)r r&rrz0Starting head extraction for {count} custom URLsrrrrrrrrrrrcK D]Z}|vrjddd|idjrn,j|j|d{\j y7#j wxYww)z%Producer to feed URLs into the queue.rrrVrrN)rrrrr)rVrrrrrrEs r$rz6AsyncUrlSeeder.extract_head_for_urls..producers $)Cd{ '+J*/:"G !((*HHSM))C.(()!!#)!!#s/B AA7A5A7$B 5A77B  B c K tjjdd{} j ||ddjxsdjjjxsdj d{j!7s#tj$r%j rj rYyYwxYw7R#t$rJ}jdd |t|d d |j|d it|dYd}~d}~wwxYw#j!wxYww)z&Worker to process URLs from the queue.T?rNFr)rextractrrrrrrrz$Failed to process URL {url}: {error}rVrrrfailed)rVstatus head_datar)rrr1rrrr rrrrr1r rrmrwr)r rVrr7rrrrs r$rz4AsyncUrlSeeder.extract_head_for_urls..workersB ' 0 0c JJC &..X" $ ' & 7%$ll(.(>(>'-'<'<'F(.(C(C)   ,OO%CK++$++-%++- ! IIg'M-03q6%B TOO""*%'!$Q % OO%s{E (B$B"B$AC! C C!E "B$$3CE CE C!! D4*AD/*D7/D44D77E  E return_exceptionsrrc3$K|]}d|v yw)rN).0r's r$ z7AsyncUrlSeeder.extract_head_for_urls..s7! A%7sc&|jddS)Nrrrrs r$r z6AsyncUrlSeeder.extract_head_for_urls..squu->'BrZr!z@Completed head extraction for {count} URLs, {success} successfulrOvalid)rsuccessr$) async_configsrr&r rrKr'rrr-r.rr/r0rr2r3rwr,cancelr4rrr5rr1anyr6)rrEr7r&rrr:r;rr producer_task worker_tasksr= worker_tasktaskr'rrrrs``` ` @@@@r$extract_head_for_urlsz$AsyncUrlSeeder.extract_head_for_urlsJs< > 4"!'F)" &L!3t9-:  ?   ""a' )%^dn o!%!(!2!263F3F!G!DND+*; <=  j1  ]]_ )+ $ $# &# &L ++HJ7  {# -A!--fWo>K    , - jjl! D KKM nnlCdCCC <KAK  K$K(K <K ?K K  K cKts|jddd|Sg}g}|D]}|jddk(rx|jdrg|j|d}|r#|j ||j |a|j |j |d}t||d <|jddk(s|j |j |d}t||d <|rh|rftj|j|j |d {}t|D]%\} }| t|kst|| |d <'|S79w) z2Apply BM25 scoring to results that have head_data.rz2BM25 scoring requested but rank_bm25 not availablerrrOrWrPrVrN) HAS_BM25rr1_extract_text_contextrw_calculate_url_relevance_scorerfloatr to_thread_calculate_bm25_score enumeraterK) rr;r7 text_contexts valid_resultsresult text_contextscorescoresis r$r5z"AsyncUrlSeeder._apply_bm25_scoringsc IIi!U[eI fN   9Fzz(#w.6::k3J#99&:MN !((6!((0!?? fUZm\E05e F,-H%0;;FLL&QV-X,1%L() 9 ]",,T-G-GWdeeF&}5 A 6s6{?05fQi0@F,- A fsCE& A!E&*E$+#E&E&c K |jj|ddd{}d|jcxkrdkrnnt|jS|jdvr)|j j d}|r t||Sy7i#t$r+}|jd d |t|d d Yd}~yd}~wwxYww)z HEAD-probe a URL. Returns: * the same URL if it answers 2xx, * the absolute redirect target if it answers 3xx, * None on any other status or network error. Frfollow_redirectsN,i-i.i/i3i4locationrzHEAD {url} failed: {err})rVerrrr) rr status_codermrVrr1rr r)rrVr'locrs r$ _resolve_headzAsyncUrlSeeder._resolve_head s kk&&sB&OOAamm)c)155z!}} 99iimmJ/"3,,P  IIg9%(Q8j  J sJC!BB0BC6B CB C!C;CCCc FKddl}tj|jj dd}|j dd|j dddj dddjd}|j d d |}|j|jd |d |d z }|jrg|se|jd d ||ddtj|d4d{} | 23d{} | j} t| |s&| ,d|vrd|dnd|d} d|jdt!| dd} d} |jd d|| ddt#| dzD]\}} |j$j'd| 4d{}|j)tj|d4d{} |j+23d{} t-j.| }|d }| j1|d!zd{t||sL|Ry7H7@6dddd{7y#1d{7swYyxYw7777M6dddd{7n#1d{7swYnxYwdddd{7y#1d{7swYyxYw#t2j4$r}|j6j8d"k(rP|t;| krB|jd#d$|| |d%dt=j>| |d{7Yd}~|jd&d'|tA|d(dd}~wtB$r'}|jd&d)|tA|d(dd}~wwxYww)*Nr ^https?://r]#r?.[/?#]+r=.jsonlrz/Loading CC URLs for {domain} from cache: {path})rrrrr'/z*.rz/*zhttps://index.commoncrawl.org/z -index?url=)safez &output=json)rrzbff]B/55c6799:<I?N!II  I  I  I+I1AN! K .I4/K 2+J7I6 J7!J2I<6I8 7I<:2J,I: -J>JN! I  I N!IN!I1%I( &I1-N!4K 6J78I<:J<J= J7J  J7J! JJ! J7$ K /J20K 5N!7K =K>K K N! K  N AM+=M>M+N! "M++ N7"NNN!c  Kt|dd}t|dd}tjdd|jd}tjdd |}t j |j jd d }|jd |d |d z } |jd |d |dz } | jr1 | j|jdddt| idd } d } d } d}|D]}dD]}|d||}|j|d {}|s(|} |jj!| ddd {}d|j"cxkrdkrnn|j$} t'| } n| sn|sz| jrjt)| ||| rE|jdddt| idt+| }|D]}t-||s|y |jddd|idg}| rX| rV|jddd| id|j/| | 23d {}|j1|t-||s'|-| rU|jddd| id|j3| 23d {}|j1|t-||s'|-d |d!} |jj!|d"dd {}d|j"cxkrdkrnn|j4j7Dcgc]D}|j9j;d#r#|j=d$d%d%j?F}}|D]?}|j3|23d {}|j1|t-||s'|-n#|jd&d'||j"d(dy |r4tA| || xsd| |jdd+tC||d,dy y #t$rYwxYw77#t$rYwxYw76b76g7Tcc}w76#t$r+}|jd&d)|t|d*dYd }~y d }~wwxYww)-a, Discover URLs from sitemaps with smart TTL-based caching. 1. Check cache validity (TTL + lastmod) 2. If valid, yield from cache 3. If invalid or force=True, fetch fresh and update cache 4. FALLBACK: If anything fails, bypass cache and fetch directly r(rr)Tr~r]rrr=Nr}sitemap_rrrzDeleted old cache format: {p}rrr)httpshttp) /sitemap.xml/sitemap_index.xmlrOrrrtruz*Loading sitemap URLs from valid cache: {p}z1Cache invalid/expired, refetching sitemap for {d}rzFound sitemap at {url}rVzhttps:// /robots.txtrqzsitemap:rrrz&robots.txt unavailable for {d} HTTP{c})rcz'Failed to fetch robots.txt for {d}: {e})rrzCached {count} URLs for {d})rr)"rrrrstriprrrnrrr-unlinkrrmr r{rr1ryrer%rCrFrY_iter_sitemap_contentrw _iter_sitemaprr splitlinesrurUrTrsrMrK)rrrWrrr=host host_saferr;old_cache_pathrHr+sitemap_contentschemesschemesuffixsmresolvedr' cached_urlsrVdiscovered_urlsrrobotsl sitemap_linesrs r$rzAsyncUrlSeeder._from_sitemapsbs"$(:B?"4)DdKvvmR077<FF8S$/ W^^-.88:2A>^^ 1VHE&JJ HYKq*OO  " %%' &"A"%s>':!;M  # F@ xs4&1!%!3!3B!77"*K"&++//+r\`/"aa!--5#5./iiO.D_.UO ! &**,z?P7P4P7R%0RP<P9P<R< R !Q *P>+8Q #A Q,Q QQ Q Q +*Q 9R PRPR!P$$ P1-R0P11R4P77R9P<<R>Q Q QQ Q>!Q94R9Q>>Rc K|jdrtj|n|}|dfd }d}g}g}tr t j d}t j ||} | jd} | jd} | r/d}| D](} || j} | s|j| *|s-| D](}||j} | s|j| *nddl mcm} |j |} | jD]4}d|j vs|j j#dd|_6| j%d}| j%d}|rDd}|D]=}|j'd}|| |jnd} | s-|j| ?|sB|D]=}|j'd}|| |jnd} | s-|j| ?|r|r؉jdddt)|id t+dt)|dz}t-j.|d}t)|}dfd }|Dcgc]}t-j0||}}||kr+j3d{}||dz }n|||kr+t-j4|ddid{y|D]}| y#t$r+}jd d |t|d d Yd}~yd}~wwxYw#t$r+}jd d|t|d d Yd}~yd}~wwxYwcc}w77w) z+Parse sitemap from already-fetched content..gzcH|syt|j}|sy|SNrrsr normalizedbase_urls r$_normalize_locz._normalize_loc' 399;7J rZFTrecoverparser1//*[local-name()='sitemap']/*[local-name()='loc']-//*[local-name()='url']/*[local-name()='loc']r-LXML parsing error for sitemap {url}: {error}rMrrNr}r .//sitemap.//urlrz4ElementTree parsing error for sitemap {url}: {error}rz2Processing sitemap index with {count} sub-sitemapsrPrrc dK j|23d{}j|d{#776n7#t$r+}jdd|t |ddYd}~nd}~wwxYwjdd{7y#jdd{7wxYww)Nr+Error processing sub-sitemap {url}: {error}rMrr)rrr rrmrHrr result_queuers r$process_subsitemapz@AsyncUrlSeeder._iter_sitemap_content..process_subsitemap%s1#'#5#5k#B22a*..q11121$C \IIg'T-83q6%JPZ\\\'**4000,**4000syB0?=9=?;?=??B A3!A.)B.A33B6B0 B  B0B-&B)'B--B0rQr Optional[str]returnrrHrm)endswithgzip decompressrr XMLParserrrrrrwr rrmxml.etree.ElementTree ElementTreeiterrrTr}rqrKr.rr/r2r1r4) rrVrer@ris_sitemap_index sub_sitemaps regular_urlsrr"sitemap_loc_nodes url_loc_nodes sitemap_elemrzloc_elemrETelemsitemaps url_entriesrurl_elemr:completed_counttotal_sitemapsrrrDitemrrrs ` @@r$rz$AsyncUrlSeeder._iter_sitemap_contentsr+.<<+>tw'G !   6''V<$(JJ/b$c! $ +Z [ $'+$(95 ,\->->?(//45 ($15,X]];(//45 / . }}T* IIK:Ddhh#'88>>##6q#9: << 5"ll84 '+$#+5#*<<#6,h>RX]]X\](//4 5 ($/5#+==#7,h>RX]]X\](//4 5  IIfR%s<'89z  KUC $5$<=J"==JP :  '#Y),s1v!>JP 2Y0 As=M"A,K+0.K+K+2 M">2L"1A9L"+AL"/L"A*M","MM"(M)M"M"M M"+ L4!LM"LM"" M+!M M"M M" M"c  K jj|ddd{}|j|jd rtj|jn |j}t|jd*fd }d}g}g}t r t#j$d} t#j&|| } | j)d} | j)d} j dd|t+| t+| dd| r/d}| D](} || j,}|s|j/|*|sF| D](}||j,}|s|j/|*|sj ddd|idnRddlmcm} |j'|} | j5D]4}d|j6vs|j6j9dd|_6| j;d}| j;d}j dd|t+|t+|dd|rDd}|D]=}|j=d }|| |j,nd}|s-|j/|?|s[|D]=}|j=d }|| |j,nd}|s-|j/|?|sj ddd|id|r|r؉j d"d#d$t+|idt?d%t+|d&z}tAjB|'d}t+|}d+fd( }|Dcgc]}tAjD||}}||kr+jd{}||dz }n|||kr+tAjF|d)did{y|D]}| y7#tj$r6}j dd||j jddYd}~yd}~wtj$r+}j dd |t|d dYd}~yd}~wt$r+}j d d |t|d dYd}~yd}~wwxYw#t$r+}j d d|t|d dYd}~yd}~wwxYw#t$r+}j d d!|t|d dYd}~yd}~wwxYwcc}w7s7Fw),NrTrrrz1Failed to fetch sitemap {url}: HTTP {status_code}rVryrrz-Network error fetching sitemap {url}: {error}rMrz0Unexpected error fetching sitemap {url}: {error}rcH|syt|j}|sy|Srrrs r$rz4AsyncUrlSeeder._iter_sitemap.._normalize_locQrrZFrrrrrzYParsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discovered)rV sitemap_countr,zvNo entries found inside tags for sitemap {url}. The sitemap might be empty or use an unexpected structure.rVrrrrrrzeElementTree parsed sitemap {url}: {sitemap_count} sitemap entries, {url_count} url entries discoveredrzrrz>Processing sitemap index with {count} sub-sitemaps in parallelrrrrc K jddd|idj|23d{}j|d{#776n7#t$r+}jdd|t |ddYd}~nd}~wwxYwjdd{7y#jdd{7wxYww) NrzProcessing sub-sitemap: {url}rVrrrrrM)rrrr rmrs r$rz8AsyncUrlSeeder._iter_sitemap..process_subsitemaps 1II!@%Q\I]cmo$(#5#5k#B22a*..q11121$C \IIg'T-83q6%JPZ\\\ '**4000,**4000sC(AAAAA A AAAAB( B !BB(B  B( C!B$"C(C=C>CCrQrr)$rr1rrrrrry RequestErrorrmr rrrrerVrrrrrrKrrrwrrrrrTr}rqr.rr/r2r4) rrVr'rr@rrrrrr"rrrrzrrrrrrrr:rrrrrDrrrrs ` @@r$rzAsyncUrlSeeder._iter_sitemap=s0 kkooc2oMMA   .1\\%-@tqyy)aiiquu: !   + 6''V<$(JJ/b$c! $ +Z [  o"),->)?%(%7 # %'+$(95 ,\->->?(//45 ($15,X]];(//45( %U$)3< * " / .. }}T* IIK:Ddhh#'88>>##6q#9:  << 5"ll84  {"),X%(%5 # '+$#+5#*<<#6,h>RX]]X\](//4 5($/5#+==#7,h>RX]]X\](//4 5 ( %U$)3< * "  IIf^%s<'89z  K UC $5$<=J"==JP b  '#Y),s1v!>JP @- 0 AsT!OOOA#T!BR9.R(,R T 2SB$S8AS<,S(A*T"S<4TTT&T>T?TOR ',PTR .!QT R  !RTR  T S!R=8T=ST S9!S4/T4S99 TTc K| r)|j|r|jddd|idy|rdnd} t|dr |js.|j | |d{} | r|j | y|r|jdd d|id|j ||d{\} }}| rd nd }|j| rd nd d|j|xs|dd| r"tjt|d{ni}|xs|||d}no|rg|jddd|id|j|d{} | rd nd }|j| rd nd d|j|dd||id}n|did}|s|r|j| ||d{|j |y7U777z7 w)Nrz Filtered out nonsense URL: {url}rVrrrrrzFetching head for {url}rW not_validrrzHEAD {status} for {final_url})rO final_url)rVrOrPzPerforming live check for {url}zLIVE CHECK {status} for {url})rOrVunknown) _is_nonsense_urlrr*rrrw _fetch_headupperrrfrr{r)rrVr rrLrrrrrr cache_kindcachedokrfinalrOrPrs r$r zAsyncUrlSeeder._validates t44S9 IIgA#S\z  ; &VF g&4::??:s;;F'  IIg8SB"'1  3$($4$4S'$BBOBe "W F IIf 3R(. U\cRXb  dGIg// TBBBbI| &E  IIg@SJ"'1  3))#..B "W F IIf 3R(. sC  U6CE 92FE 7//*c59 9 9G<C C/ :s]AG!F6"AG(F9)AGF<>GF>AGG G9G<G>GGc K |jj||dddd{}|jy7#tj$r+}|j dd|t |dd Yd}~y d}~wtj$r6}|j dd ||jjd d Yd}~y d}~wt$r+}|j dd|t |dd Yd}~y d}~wwxYww)Nz bytes=0-0identity)RangeAccept-Encoding)rrTrz+HEAD check network error for {url}: {error}rMrrFz5HEAD check HTTP status error for {url}: {status_code}rrz5Unexpected error during HEAD check for {url}: {error}) rrrrrrrmrrryr )rrVrr'rs r$_head_okzAsyncUrlSeeder._head_ok"s kk&&sG9DYc/d'ffA   f!!  IIgL%(3q6:   L$$  IIgV%(9O9OPV`  b  IIgV%(3q6:   L s_C:$A>AC:AC7!A94C:9C7,C;C: C7 !C2-C:2C77C:c Kt|dzD] } |jjd||ddid4d{}|jdvr|jj d}|r@t ||}|jd d |j|d d  dddd{|jdd|j|jdd ddt|jfcdddd{cSd|jcxkrdksYn|jdd|j|jdd ddt|jfcdddd{cSt} |j|23d{} | j| | j} d| vst| |k\s=|jd{|jj ddj} | dk(r| dddk(rt!j"| } nO| dk(r$t$r| dddk(rt'j"| } n&| dvr"|jd d| |jdd | jj+d}|d"k(r7|jd d#d$|jid t| d%kr| n| dd%}n| d|d&z} |j-d'd(}d,|t|jfcdddd{cS|jdd.||d/d dd|fS777I777q6q#t($r7} |jdd |j| t| d!d Yd} ~ d} ~ wwxYw#t($rG} |jdd)|jt| d*d |j-d+d(}Yd} ~ d} ~ wwxYw7#1d{7swYnxYw#t.j0$r2} |jd d-|t| d*d dd|fcYd} ~ cSd} ~ wwxYww)0NrrrrF)rrrsrvLocationrz,Redirecting from {original_url} to {new_url}) original_urlnew_urlrrrz>Redirect status {status_code} but no Location header for {url})ryrVr]rtiz=Non-success status {status_code} when fetching head for {url}szContent-EncodingrsbrrRsl >rrz#Skipping bogus {encoding} for {url})encodingrVz3Decompression error for {url} ({encoding}): {error})rVrrrPz.No tag found in initial bytes of {url}rVi(rr\r3z0Failed to decode head content for {url}: {error}rMzlatin-1Tz+Fetch head network error for {url}: {error}z2Exceeded max redirects ({max_redirects}) for {url}) max_redirectsrV)r3rrryrr1rrrVrm bytearray aiter_bytesextendrurKacloserr HAS_BROTLIbrotlir rqdecoderr)rrVrr  max_bytes chunk_sizer=r'rwbufchunklowencridx html_bytesrs r$rzAsyncUrlSeeder._fetch_head5s}Q'_ &A^ & ;;--#*:&+. V2V2}}(AA#$99==#<#")#x"8C IIg/]>?eePS-TZd&f$#V2V2V2&!IIi1q=>]]STSXSX-Y_i&k$)"c!%%j#8-V2V2V24 1==636 )-l9:quu)U[e"g$b#aee*4;V2V2V2>$+C'(}}Z'@""e 5)!iik%,CI0E"#((*,,!))--(:B?EEGC&=S!W -C"&//#"6C D[ZCGGZJPb#~% &sQ &PL>PAO+ P&M'P+Q ,AO+/ P;M<PQ AO+ P)M*P.Q 1O+ MM M4O+ O+M /O+ A2M?A#O+#N5O+ PO)P!Q >PPPP M O+O+ N,NO+NO+ O&=O!O+!O&&O+)P+O= 1O42O= 9PQ Q%Q:Q;Q QQ cg}|jdr|j|d|jdi}dD](}|j|s|j||*dD](}|j|s|j||*dD](}|j|s|j||*dD](}|j|s|j||*|jdgD]}t|tsdD]^}||vst||tr|j||0t||t sD|j d ||D`d |vs|t|d t s|d D]G}t|tsd D]/}||vst||ts|j||1Id jtd |S)z9Extract all relevant text from head metadata for scoring.r^r`) descriptionkeywordsauthorsubjectsummaryabstract)zog:titlezog:descriptionz og:site_namez article:tag)z twitter:titleztwitter:descriptionztwitter:image:alt)zdc.titlezdc.descriptionz dc.subjectz dc.creatorrb)rcheadlinerrrc36K|]}|r t|ywr)rm)rTrs r$rUz7AsyncUrlSeeder._extract_text_context..s .Q264/2$i.Qsz@graph)rcr r N) r1rwrlrArmlistr r,filter)rrP text_partsr`r"rbfieldrs r$rcz$AsyncUrlSeeder._extract_text_contexts   == !   i0 1}}VR(Z -Cxx}!!$s), - Q -Cxx}!!$s), - Q -Cxx}!!$s), - N -Cxx}!!$s), -  mmHb1 CF&$'XQE%fUmS9&--fUm<'u t<&--.Q:@-.QQ Qv%*VH5Et*L &x 0C%dD1)LC#(D=ZU S5Q$.$5$5d5k$BCC C(xxtZ011rZc|j}|j}ddlm}||}|jj dd}|j j d}|jd} |jdD cgc]} | s|  } } |j} g} | rC| jdD]/}d|vs|jdd \}}| j||g1| | z| z}g}|j}|D]>}|j}||vr|jd )||vs.|jd @|D]}g}|D]r}|j}||vr,t|t|z }|jd |zC||vsHt|t|z }|jd |zt|s|jt|dd}dj|j}t|dk\r^t|dk\rP||}||}|r>|r.get_ngrams..sAqtAac{Asr)rr3rK)rrr.s``r$ get_ngramszAAsyncUrlSeeder._calculate_url_relevance_score..get_ngramss%AE#d)A+a-,@AA ArZr"?rT)r#)r)ru urllib.parser)netlocr3rrsrTrr rwrKrr,r6rhr.)&rrrV query_lower url_lowerr)parsedrr domain_partsr path_parts query_params param_partsparamr"value all_partsrn query_tokenspart part_lowertoken token_scorescoverager/url_text query_ngrams url_ngrams intersectionunionjaccardweighted_score total_weightrormweight final_scores& r$rdz-AsyncUrlSeeder._calculate_url_relevance_scoreskkm IIK  *#&&vr2{{  %||C( !%C6AAa6 6||  %++C0 5%.Ls CTt}}qZ(Cs!rc3DK|]}|s|jdyw)rN)rU)rTr>s r$rUz2AsyncUrlSeeder._is_nonsense_url..xsCdts#Cs  )z /wp-adminz /wp-includesz/wp-content/uploadsz/adminz/loginz/signinz/signupz /registerz /checkoutz/cartz/accountz/profilez/searchz/404z/errorz/.gitz/.svnz/.hgz/cgi-binz/scriptsz /includesc3&K|]}|v ywrrS)rTncprs r$rUz2AsyncUrlSeeder._is_nonsense_url..s8ssd{8c3&K|]}|v ywrrS)rTrWr4s r$rUz2AsyncUrlSeeder._is_nonsense_url..s`w)#`rS)z?print=z&print=z/print/z_print.r)rz/enz/dez/frz/esz/itF) rur1r)rrr[rTrKrs) rrVr)r5 utility_filesr7non_content_pathsrr4s @@r$rzAsyncUrlSeeder._is_nonsense_url1s IIK  *#{{  " ==N O  $--0K"L C]C CTZZ_ C C C  8&78 8 `3_` ` tzz# ! #4\(\rZc ts#|jddddgt|zS|r|sdgt|zS|jj }|Dcgc] }|jj "}}t d|Drdgt|zS ddlm}||}|j|}t|dk(rgSt|} t|} | | k(rd gt|zS|D cgc] } | | z | | z z } } | Scc}wcc} w#t$r9} |jd d d t| id dgt|zcYd } ~ Sd } ~ wwxYw)z4Calculate BM25 scores for documents against a query.rz/rank_bm25 not installed. Returning zero scores.rrrc38K|]}t|dk(yw)rN)rK)rTrs r$rUz7AsyncUrlSeeder._calculate_bm25_score..s7s3x1}7sr) BM25Okapir0rz&Error calculating BM25 scores: {error}rN) rbrrKrurTall rank_bm25rY get_scoresr.rr rm)rr documentsr=rtokenized_docsrYrrn min_score max_scorermnormalized_scoresrs r$rgz$AsyncUrlSeeder._calculate_bm25_scores IILR\  ^53y>) )I53y>) ){{}**, 9BC##))+++-CC 77 753y>) ) * +^,D__\2F6{a F IF II%us6{**]c cSX%)"3 I8M!N c  c$ $7D2!d * IIgG%s1v.J  @53y>) ) *sB%D(.D%)D%D%D D% D%% E'..E"E'"E'cK|jrD|jr7|jjd{|jdddyyy7w)z#Close the HTTP client if we own it.NrzClosed HTTP clientrr)rrr rrs r$closezAsyncUrlSeeder.closesI   ++$$& & & IIg3I D"-  &s6AAAcK|Sw)zAsync context manager entry.rSrcs r$ __aenter__zAsyncUrlSeeder.__aenter__s  sc@K|jd{y7w)zAsync context manager exit.NF)rd)rexc_typeexc_valexc_tbs r$ __aexit__zAsyncUrlSeeder.__aexit__sjjl s c K|jjrtj|jjjz |j j krI|jddd|jid|jjjS|jdddtid tj4d{}|jtd d{}|j|jd d }|jj!||jd d d|id|cdddd{S77q7 #1d{7swYyxYw#tj"$r&}|jdddt%|idd}~wtj&$r1}|jddd|j(j*idd}~wt,$r&}|jdddt%|idd}~wwxYww)Nrz*Loading latest CC index from cache: {path}rrrz-Fetching latest Common Crawl index from {url}rVrqrKridrXz4Successfully fetched and cached CC index: {index_id}rrz-Network error fetching CC index info: {error}z0HTTP error fetching CC index info: {status_code}ryz0Unexpected error fetching CC index info: {error})rr-rrrrr6r read_textrs COLLINFO_URLrrr1rr/ write_textrrmrrryr )rrjrrs r$r+zAsyncUrlSeeder._latest_indexs  ' ' )tyy{4;P;P;U;U;W;`;`/`dhdldldzdzd|.| IIfJ$d&;&;<*  N((224::< < &I.J  @ ((*  a%% b%99""$ffhqk$'%%005 )%[",c!2 D   9    !!  IIgN%s1v.J  @ $$  IIgQ+QZZ-C-CD*  V   IIgQ%s1v.J  @  sCIF.E=/F2F E?AF+ F7F8F<I=F?FFF F FFIFI+!G  I",H I!H;;II) rrrzOptional[httpx.AsyncClient]rzOptional[AsyncLoggerBase]rz"Optional[Union[str, pathlib.Path]]rzOptional[Union[str, Path]])r)rrmrrmrrmrr)rrmrVrmrr)rrmrVrmrzOptional[Dict[str, Any]])rrmrVrmr@Dict[str, Any]rNone)rrmr7'SeedingConfig'rr%)rCz Sequence[str]r7rtrzDict[str, List[Dict[str, Any]]])Nrqr) rE List[str]r7zOptional['SeedingConfig']r&intrrvrr%)r;r%r7rtrr%)rVrmrr)rrmrWrmrbool)F)rVrmrebytes)rVrm)NNrT)rVrmr r%rrwrLrwrrvrrwrrrzOptional[float]rrmrrw)rVrmrrvrrw)rii) rVrmrrvr rvrrvrrv)rPrrrrm)rrmrVrmrre)rVrmrrw)rrmr]rurz List[float])rrm)__name__ __module__ __qualname____doc__TTLrrrrrrErFr`r5r{rrrrr rrrcrdrrgrdrfrkr+rSrZr$rrsF.2,0=A158 8,8* 8 ; 8/8B<4 y?y?*y?-y?v!! ! ) !L-1 ^^*^ ^  ^  ^@ D<5pp[dgRl`\`W]044!%4034>B4KX4)84QT4*.4l. l ll l  l  l^32jV%p^@,*^E  rZr)r!rxrrr) r; pathlib.Pathr<rvr=rwr>rrrw)r;r~rru) r;r~rErurHrmr+rrrs)rVrmrWrmrrw)rrmrrr)>r| __future__rrrrrior/rrrrrrrrtypingrr r r r r rr1rrrrSlxmlrrorr ImportErrorrrr[rb async_loggerrrrrYrror}rrr|r~rryrr%rCrFrMrYrrrSrZr$rs #   22GGG' & DJH7!,= Q 2::|DD bjjBBDDI BJJ.rtt < 2::Dbdd L &&* ////# /  /d    #    ,SFVgg D J Hs6$E3E':E4E$#E$'E10E14E>=E>