/ir ddlZddlmZmZddlmZmZmZmZm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlmZddlmZmZddlmZmZmZmZmZddlZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+ddl,Z,dd l-m.Z.ddl/Z/ddl0Z0ddl1Z1ddl2Z2ddl3m4Z4ddl5Z5ddl m6Z6m Z7ddl8Z8ddl9Z9ddl:m;Z;ddlm?Z?ddl@mAZAddl!mBZBddlCmDZDddlEmFZFddlGZGddlHZIddl*m+Z+mJZJmKZKmLZLmMZMmNZNmOZOddlPZPddl:mQZQddl Z eQjZSdZTeTeQ_R dde)eUdeVdeVdeWde%e&eUge$eUfde'eUddff d ZX dd!eBeUd"eVdeVd#eWd$e&de$eUf d%ZYGd&d'ZZGd(d)Z[Gd*d+e\Z]e^gdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddgd,zZ_hd-Z`d.eUdeaeUfd/Zb dd2eUd3eUd4eVd5ecd6ecdeUf d7Zdd8Zed9Zfd:ZgdeUfd;Zhd<Zid=Zjd>Zkd.eUdeUfd?Zld@ZmddAZnedfdBZoedfdCeUdDeUdEeVdFeUde"eUe#ff dGZpddHZqddIZrdJZsdKZtdLZu ddMZv ddNZweddfdOZxddPZydQZz ddCeUdReadSeUdTeUdeaf dUZ{dVZ|dWZ}dXZ~dYZd0d0d1dd1ddZd[eUd\eUfd]Zdd^Ze=d_`ddaZdbZdDeUd\eUde"eUe$e"eUeUfffdcZdCeUdeUfddZdCeUdeeUdecfdfZdgeaeUdeaeUfdhZdiZdjeUdeUfdkZdleUdeUfdmZdneUde"eUeUffdoZdpZddqeVfdrZdsZddtZGdudvZddwZdxZdyZdzeUd{eUd|eUd\eUdef d}Zd~eUdCeUdeUdeUdeUdedeWfdZ dde%eWde%eWdecdecdeWf dZ dde$eUde%e"deUdeVdeIj.f dZ dde$eUde%e"deUdeVdeIj.f dZdeIj.deIj.deWfdZdeIj.deIj.deWfdZdeWfdZdeWfdZde(eWeWeWffdZde"eUe&fde"eUeUffdZy)N)ThreadPoolExecutor as_completed) BeautifulSoupCommentelementTagNavigableString)PROMPT_EXTRACT_BLOCKS)array) html2textCustomHTML2Text)MIN_WORD_THRESHOLD$IMAGE_DESCRIPTION_MIN_WORD_THRESHOLDIMAGE_SCORE_THRESHOLDDEFAULT_PROVIDERPROVIDER_MODELS)gaierror)Path)DictAnyListOptionalCallable GeneratorTupleIterableurljoin) InvalidSchemawraps)etreehtml)RobotFileParser) lru_cache)version) __version__)Sequence)chain)deque)rurlparse urlunparse parse_qsl urlencodequoteunquote)RuleLinecd|jvsd|jvs|jdvr|jjdd}tj|jdd}d|z}|j dr|ddd z} t tj ||St||S#tj$rt||cYSwxYw) N*%2A)r4r5z\*z.*^z\$$) pathreplacereescapeendswithboolmatcherrororiginal_applies_to)selffilenamepatterns J/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/crawl4ai/utils.pypatched_applies_torF;s TYY%499, \0I""5#.w7#++E48www   5 !Sb\C'76rxx23 3 dH --hh6%dH5 56s B66CC documentschunk_token_thresholdoverlapword_token_rate tokenizerreturnc#Kt}t}d}|D]}|r||} dgt| z} n|j} |gt| z} |j| |j| |t | z }||k\ssg} g} d} |rT|d}| |z|kDrnF| |z } | j |j | j |j |rT| s>| j |j | j |j d}d}t| D]}||z|kDrn ||z }|dz }|dkDrE| | d}| | d}|jt||jt|||z }|t | z}dj|r| dt| |z n| ||k\rI|rdj|yyw)a Efficiently chunks documents into token-limited sections with overlap between chunks. Args: documents: Iterable of document strings chunk_token_threshold: Maximum tokens per chunk overlap: Number of tokens to overlap between chunks word_token_rate: Token estimate per word when not using a tokenizer tokenizer: Function that splits text into tokens (if available) Yields: Text chunks as strings ?rr N ) r+lensplitextendsumappendpopleftreversed extendleftjoin)rGrHrIrJrK token_queuecontribution_queuecurrent_token_countdoctokens contributions chunk_tokens chunk_contrib chunk_total next_contrib overlap_total overlap_idxcontriboverlap_tokensoverlap_contribs rEchunk_documentsriLs8('K8j s^F ECK/MYY[F,-F ;M 6"!!-0s=11"%::LMK%1!4 -0EE|+ $$%7%?%?%AB##K$7$7$9:%!$$%7%?%?%AB##K$7$7$9: MK#M2 ! 7*W4( q  !Q!-{lm!<"/ ">&&x'?@"--h.GH#}4# 3}#5 5 ((;<(F\):;)FG\hi iS"%::8jvhh{##sB G1AG1*C+G1G1docs target_sizeword_token_ratiosplitterc|xstj}td}g}d}|D]K}||} tt | |z} | s%|j | |j | || z }M|sgSt d||zdz |z} t| D cgc]} g} } d}d}tj|D]b} ||k\rB|| dz kr:|dkDr.| || d}|dz }| |j|t |}n|dz }d}| |j | |dz }d| Dcgc]}|sdj|c}Scc} wcc}w)a` Merges a sequence of documents into chunks based on a target token count, with optional overlap. Each document is split into tokens using the provided splitter function (defaults to str.split). Tokens are distributed into chunks aiming for the specified target size, with optional overlapping tokens between consecutive chunks. Returns a list of non-empty merged chunks as strings. Args: docs: Sequence of input document strings to be merged. target_size: Target number of tokens per chunk. overlap: Number of tokens to overlap between consecutive chunks. word_token_ratio: Multiplier to estimate token count from word count. splitter: Callable used to split each document into tokens. Returns: List of merged document chunks as strings, each not exceeding the target token size. Irr NrP) strrRr intrQrUmaxranger* from_iterablerSrY)rjrkrIrlrm token_counts all_tokens total_tokensr]r^count num_chunks_chunks curr_chunk curr_sizergchunks rE merge_chunksrs.$399H:L"$JL"#CK"223     &   f % E !L "  Q 3a7KGHJ+0+<=ar=F=JI%%j1   # Z!^(C{!' !3WHI!>a z")).9/ a  z!!&)Q  *0 95CHHUO 99+>* :s E'E /E c$eZdZdZdZdZdZy)VersionManagercdtjdz |_|jdz |_y)N .crawl4aiz version.txt)rhomehome_dir version_filerBs rE__init__zVersionManager.__init__s% k1  MMM9c|jjsy tj|jj j S#t $r }Yd}~yd}~wwxYw)z*Get the version recorded in home directoryN)rexistsr'parse read_textstrip Exception)rB_exs rEget_installed_versionz$VersionManager.get_installed_versionsV  '') ==!2!2!!D!D!FG G  s:A A+&A+cV|jjtjy)z2Update the version file to current library versionN)r write_textr(rs rEupdate_versionzVersionManager.update_versions $$[%<%<=rc~|j}tjtj}|duxs||kS)z/Check if database needs update based on versionN)rr'rr()rB installedcurrents rE needs_updatezVersionManager.needs_updates9..0 -- 7 78D 7I$77rN)__name__ __module__ __qualname__rrrrrrErrs:>8rrcjeZdZdZddZdZdedeeeffdZ dedefd Z dd ed edefd Z d Z dZ y) RobotsParseri: NcR|xs)tjjtdd|_|xs |j |_tj|jdtjj|jd|_|jy)NrrobotsTexist_okzrobots_cache.db) osr9rYget_home_folder cache_dir CACHE_TTL cache_ttlmakedirsdb_path_init_db)rBrrs rErzRobotsParser.__init__si"\bggll?3DkS[&\"4dnn DNNT2ww||DNN4EF  rctj|j5}|jd|jd|jddddy#1swYyxYw)NzPRAGMA journal_mode=WALa CREATE TABLE IF NOT EXISTS robots_cache ( domain TEXT PRIMARY KEY, rules TEXT NOT NULL, fetch_time INTEGER NOT NULL, hash TEXT NOT NULL ) z=CREATE INDEX IF NOT EXISTS idx_domain ON robots_cache(domain)sqlite3connectrexecuterBconns rErzRobotsParser._init_dbsZ __T\\ * Zd LL2 3 LL  LLX Y Z Z Zs 4AA&domainrLc"tj|j5}|jd|f}|j }|s dddy|\}}}|t j |z |j kfcdddS#1swYyxYw)z+Get cached rules. Returns (rules, is_fresh)zASELECT rules, fetch_time, hash FROM robots_cache WHERE domain = ?N)NF)rrrrfetchonetimer)rBrrcursorresultrules fetch_timerzs rE_get_cached_ruleszRobotsParser._get_cached_ruless __T\\ * Fd\\S F__&F" F F$* E:q499;3t~~EE F F Fs'B+BBcontentc tj|jj}t j |j 5}|jd|f}|j}|r|d|k7r1|jd||ttj|fdddy#1swYyxYw)z7Cache robots.txt content with hash for change detectionz.SELECT hash FROM robots_cache WHERE domain = ?rzINSERT OR REPLACE INTO robots_cache (domain, rules, fetch_time, hash) VALUES (?, ?, ?, ?)N) hashlibmd5encode hexdigestrrrrrrqr)rBrrhash_valrrrs rE _cache_ruleszRobotsParser._cache_rules%s;;w~~/0::< __T\\ * d\\@ F__&FVAY(2 .Wc$))+&6A    s AB99Curl user_agentctK t|}|j}|sy |j|\}}|s |jxsd}|d|d} t j 4d{} | j| dd4d{} | jd k(r+| jd{}|j||n" dddd{dddd{ydddd{dddd{|syt} | j|j| jsy| j||S#t$r }Yd}~yd}~wwxYw777777#1d{7swYxYw7#1d{7swYxYw#t$r }Yd}~yd}~wwxYww) a2 Check if URL can be fetched according to robots.txt rules. Args: url: The URL to check user_agent: User agent string to check against (default: "*") Returns: bool: True if allowed, False if disallowed by robots.txt TNhttpz://z /robots.txtF)timeoutssl)r,netlocrrschemeaiohttp ClientSessiongetstatustextrr%r splitlinesmtime can_fetch) rBrrparsedrrris_freshr robots_urlsessionresponseparsers rErzRobotsParser.can_fetch9s c]F]]F 008x 0& &xs6(+> "002((g&{{:qe{L((PX#??c1*2--/$9E --fe<#' (((((((((!" U%%'(||~ C00G  (($9((((((((((  s#F8EF80F"&E)'F"*F E,F #E6*E.+E6 F E0F  F"E2F"#F8$ F /E40F 4 F"?F F"AF8 E&F8!E&&F8)F",F .E60F 2F"4F 6F <E?=F F  F" FF FF"" F5+F80F55F8ctj|j5}|jddddy#1swYyxYw)z#Clear all cached robots.txt entrieszDELETE FROM robots_cacheNrrs rE clear_cachezRobotsParser.clear_cacheos6 __T\\ * 5d LL3 4 5 5 5s ;Actj|j5}tt j|j z }|j d|fdddy#1swYyxYw)z&Remove only expired entries from cachez-DELETE FROM robots_cache WHERE fetch_time < ?N)rrrrqrrr)rBr expire_times rE clear_expiredzRobotsParser.clear_expiredtsW __T\\ * Zddiik*T^^;K LLH;. Y Z Z Zs >A''A0NN)r4)rrrrrrrptupler>rrrrrrrrErrsj I ZFFc4i0@F 3(41341C41$41l5 Zrrc eZdZy)InvalidCSSSelectorErrorN)rrrrrrErr{srr>B–—•…™⇒≈≤≥►▼!=&&*=+=---=/=<<=>>>?.?:??||<=>===r4rz©«®»→!"#r8%&()+,-./:;<=>?@[\]r6`{|}~++, ==rcg}td}d}t|}||kr||}t|}|dkr6t|r-|r|j |j td}np||dz krW|||dzz}|t vr2|r*|j |j td}|dz }n#|j |n|j ||dz }||kr|r|j |j |S)Nurr )r rQordSPLITSrU tounicodeHTML_CODE_CHARS)rrworditext_lencharo two_charss rEadvanced_splitr0s F :D A4yH h,Aw I s7vay dnn./Sz A tAE{*IO+MM$.."23 :DQ D! KK  Q+ h,.  dnn&' MrTFmessagetypewidth add_newlines double_linec>dddddd}|j|j|d\}}}dd d } |rd nd } | | \} } } }}}g}|jd }|r|d|dj}t j ||dz }|j |jd |ddD]o}|jrLt j d|j|dz }|j |jd _|jdq| |dz z}d|d| ||d|d g|Dcgc]%}d|d| d|d|d|dz dd|d|d| d|d'c}d|d|||d|d }d j|}|rd |d }|Scc}w)a Create a styled message box with colored borders and formatted text. How it works: 1. Determines box style and colors based on the message type (e.g., info, warning). 2. Wraps text to fit within the specified width. 3. Constructs a box using characters (single or double lines) with appropriate formatting. 4. Adds optional newlines before and after the box. Args: message (str): The message to display inside the box. type (str): Type of the message (e.g., "info", "warning", "error", "success"). Defaults to "info". width (int): Width of the box. Defaults to 120. add_newlines (bool): Whether to add newlines before and after the box. Defaults to True. double_line (bool): Whether to use double lines for the box border. Defaults to False. Returns: str: A formatted string containing the styled message box. )yellow bright_yellowu⚠)blue bright_blueuℹ) lightblack bright_blacku⋯)green bright_greenu✓)red bright_red×)warninginfodebugsuccessr@rC)u─u│u┌u┐u└u┘)u═u║u╔u╗u╚u╝)singledoublerGrF rPr)r3r N rrz[/z] rrz][) rlowerrRrtextwrapfillrSrUrY)r1r2r3r4r5styles border_color text_colorprefix box_chars line_styleh_linev_linetltrblbrformatted_lines raw_lines first_line wrapped_firstlinewrappedhorizontal_lineboxrs rEcreate_box_messagercs':6.63, F(.zz$**,v'O$L*f=<I)hJ%.z%:"FFBBO d#Ixq1!3!3!5 67  j B }22489abM +Dzz|"--"TZZ\N(;519M&&w}}T':;&&r*  + *O L>2$/t2l^1E (  ~Qvha |2d1U1WIP+5Fb TVWcVddeflemmop|o}}~    L>2$/t2l^1E  CYYs^FfXR M s*Fctj}tdz }td|dz}t |dz }t ||S)a Calculate the optimal semaphore count based on system resources. How it works: 1. Determines the number of CPU cores and total system memory. 2. Sets a base count as half of the available CPU cores. 3. Limits the count based on memory, assuming 2GB per semaphore instance. 4. Returns the minimum value between CPU and memory-based limits. Returns: int: The calculated semaphore count. @r r)r cpu_countget_system_memoryrrrqmin)rf memory_gb base_countmemory_based_caps rEcalculate_semaphore_countrlsI I!#w/IQ Q'J9q=) z+ ,,rc tj}|dk(rYtdd5}|D]=}|jdst |j ddzccdddS dddy|dk(r@d dl}|jgd jd }t |jS|d k(rd dl jj} jG fd d j}|} j||_|j# j%||j&St)d#1swYyxYw)af Get the total system memory in bytes. How it works: 1. Detects the operating system. 2. Reads memory information from system-specific commands or files. 3. Converts the memory to bytes for uniformity. Returns: int: The total system memory in bytes. Raises: OSError: If the operating system is unsupported. Linuxz /proc/meminforz MemTotal:r NDarwinr)sysctlz-nz hw.memsizeutf-8Windowsc eZdZdWjfdWjfdWfdWfdWfdWfdWfdWfd Wfg Zy ) )get_system_memory..MEMORYSTATUSEXdwLength dwMemoryLoad ullTotalPhys ullAvailPhysullTotalPageFileullAvailPageFileullTotalVirtualullAvailVirtualullAvailExtendedVirtualN)rrrc_ulong_fields_) c_ulonglongctypessrEMEMORYSTATUSEXrvLsZV^^,0--#[1#[1"K0"K0*K8 HrrzUnsupported operating system)platformsystemopen startswithrqrR subprocess check_outputdecoderrwindllkernel32r StructuresizeofrwGlobalMemoryStatusExbyrefryOSError) rmemr_routputrr memoryStatusrrs @@rErgrg+s= __ F  /3 ' 73 7??;/tzz|A/$66 7 7 7 7 7 8 ((()GHOOPWX6<<>"" 9 ==))((  V-- &' & n = %%fll<&@A(((455C 7 7sE E*EEc Ptjjtjdtjdt j d}tj |dtj |ddtj |dd|S)at Get or create the home folder for Crawl4AI configuration and cache. How it works: 1. Uses environment variables or defaults to the user's home directory. 2. Creates `.crawl4ai` and its subdirectories (`cache`, `models`) if they don't exist. 3. Returns the path to the home folder. Returns: str: The path to the Crawl4AI home folder. CRAWL4_AI_BASE_DIRECTORYrTrz/cachez/models)rr9rYgetenvrrr) home_folders rErras'',, & II0$))+ >   KKK d+KK;-v&6KK;-w'$7 rcKdddd}|j|}|std|t}tjj ||j d}tjj|r&t|d5}|jcdddSd d l m }|4d{}|j|j|jd}|j |vr+td d j |j!t}||j j"}|std |ttjj ||j dd5}|j%|ddd|cdddd{S#1swY xYw7#1swY,xYw7!#1d{7swYyxYww)aReturns the browser executable path using playwright's browser management. Uses playwright's built-in browser management to get the correct browser executable path regardless of platform. This ensures we're using the same browser version that playwright is tested with. Returns: str: Path to browser executable Raises: RuntimeError: If browser executable cannot be found chromiumfirefoxwebkit)rrrzUnsupported browser type: z.pathroNr)async_playwrightz&Invalid browser type. Must be one of: r!z'Browser executable not found for type: w)r RuntimeErrorrrr9rYrLrrreadplaywright.async_apirrrr ValueErrorkeysexecutable_pathwrite) browser_type browser_typesr path_filefrpbrowsers browser_paths rEget_chromium_pathrzsM !$$\2L 7 ~FGG"#K [\-?-?-A,B%*HII ww~~i )S ! Q668  6!Q yyhh     x /88==?9S8TU  &'  2 2 45EE !HWX X "'',,{|/A/A/C.DE,JKS Q "UV GGL ! "+   $ " "%syBG3G G3;G <G3?CGG% G. G3:G;G3G  G3G GG3G0$G' %G0,G3chtj|}t|d}|j}|S)z Beautifies an escaped HTML string. Parameters: escaped_html (str): A string containing escaped HTML. Returns: str: A beautifully formatted HTML string. html.parser)r$unescaperprettify) escaped_htmlunescaped_htmlsoup pretty_htmls rE beautify_htmlrs1]]<0N  7D--/K rc|jdr$|jdr|ddj}g}d}d}t|D]?\}}|dk(r |dk(r|}|dz }|dk(s|dz}|dk(s)|j |||dzAg}g}|D])} t j |} |j | +||fS#t j$r|j |YVwxYw)a Splits a JSON string which is a list of objects and tries to parse each object. Parameters: json_string (str): A string representation of a list of JSON objects, e.g., '[{...}, {...}, ...]'. Returns: tuple: A tuple containing two lists: - First list contains all successfully parsed JSON objects. - Second list contains the string representations of all segments that couldn't be parsed. rrr rrr)rr=r enumeraterUjsonloadsJSONDecodeError) json_stringsegmentsdepth start_indexr+r-parsed_objectsunparsed_segmentssegmentobjs rEsplit_and_parse_json_objectsrsc"{';';C'@!!B'--/ H EK[)B4 3;z QJE S[ QJEz K!a% @ABN. .**W%C  ! !# &. , ,,## .  $ $W - .s&B??$C&%C&cN|}|jddjdd}|S)a< Sanitize an HTML string by escaping quotes. How it works: 1. Replaces all unwanted and special characters with an empty string. 2. Escapes double and single quotes for safe usage. Args: html (str): The HTML string to sanitize. Returns: str: The sanitized HTML string. r\"'z\')r:)r$sanitized_htmls rE sanitize_htmlrs0 N$++C7??UKN rc. |sy|jddjdS#t$r:}td||jddjdcYd}~Sd}~wwxYw#t$r}t dt ||d}~wwxYw) z3Sanitize input to handle potential encoding issues.rKrsignoreerrorszFWarning: Encoding issue detected. Some characters may be lost. Error: asciiNzError sanitizing input: )rrUnicodeEncodeErrorprintrrrp)res rEsanitize_input_encoder s E I;;wx;8??H H! I XYZX[\ ;;wx;8??H H  I E3CF8<=1DEs<(!( A+/A& A+!A.&A++A.. B7BBc2|jdd}|jdd}|jdd}|jdd}|jd d }|jd d }|jd d}tjdd|}|S)z Escapes characters in a string to be JSON safe. Parameters: s (str): The input string to be escaped. Returns: str: The escaped string, safe for JSON encoding. rz\\rrz\b z\frHz\n z\r z\tz[\x00-\x1f\x7f-\x9f]cRdjt|jS)Nz\u{:04x})formatr&group)xs rEz$escape_json_string..4s+2D2DS^2Tr)r:r;sub)ss rEescape_json_stringrs $A #uA $A $A $A $A $A &(TVWXA Hrciddddddddd d d d d ddddddddddddddddddd d!d"d#d$d%d&d'}|Dcgc]}||j|d(f}}|D]B\}}|j|D])}|r |jn||}|j|+D|Scc}w))aN Replace inline HTML tags with Markdown-style equivalents. How it works: 1. Maps specific tags (e.g., , ) to Markdown syntax. 2. Finds and replaces all occurrences of these tags in the provided BeautifulSoup object. 3. Optionally replaces tags with their text content only. Args: soup (BeautifulSoup): Parsed HTML content. tags (List[str]): List of tags to replace. only_text (bool): Whether to replace tags with plain text. Defaults to False. Returns: BeautifulSoup: Updated BeautifulSoup object with replaced tags. bc"d|jdSNz**rtags rErz%replace_inline_tags..L2chhZr*rr+c"d|jdSNr4rrs rErz%replace_inline_tags..M1SXXJarr$c"d|jdS)N__rrs rErz%replace_inline_tags..Nrrspanc|jSNrrs rErz%replace_inline_tags..O sxxjrdelc"d|jdSNz~~rrs rErz%replace_inline_tags..PRz,rinsc"d|jdS)Nr rrs rErz%replace_inline_tags..Qrrrc"d|jdS)Nrrrs rErz%replace_inline_tags..RQsxxj?rsupc"d|jdS)Nz^^rrs rErz%replace_inline_tags..Srrstrongc"d|jdSrrrs rErz%replace_inline_tags..Ts388*B/remc"d|jdSrrrs rErz%replace_inline_tags..UsAchhZq/rcodec"d|jdSNrrrs rErz%replace_inline_tags..VazOrkbdc"d|jdSrrrs rErz%replace_inline_tags..Wrrvarc"d|jdSNrzrrs rErz%replace_inline_tags..Xrrrc"d|jdSrrrs rErz%replace_inline_tags..Yrrqc"d|jdS)Nrrrs rErz%replace_inline_tags..ZrrabbrcF|jd|jdddS)Nz (titlerKr )rrrs rErz%replace_inline_tags..[s#sxxj3777B+?*@Brcitec"d|jdSr rrs rErz%replace_inline_tags..\rrc"d|jdSr rrs rErz%replace_inline_tags..]rrc|jSrrrs rErz%replace_inline_tags..^rrc"d|jdS)Nzzrrs rErz%replace_inline_tags.._swsxxj9rc"d|jdS)Nr"rrs rErz%replace_inline_tags..`sb "-r)dfnrsmallmarkc|jSrr)ts rErz%replace_inline_tags..ds !&&r)rfind_allr replace_with) rtags only_texttag_replacementsrreplacement_datatag_namereplacement_funcreplacement_texts rEreplace_inline_tagsr$9s$ * ( * )  ,  ,  * , / ) + * * * ( B!" +#$+)9-+2GK?B""3(89:'7/""==* /C+4sxx:J3:O    - . // KsB=c ~ |syt|d}|j}|rL|j|}|std||j d}|D]} |j | |}ggd} |j ddD]{} | d } |jd d } | jd r*| | vr&| d j | | jdW| dj | | jd}|j gdD]}|j|j D]}|jdk7si|_ gggd}|j dD]8}|dj |jd|jddd:|j dD]8}|dj |jd|jddd:|j dD]8}|dj |jd|jddd:|j dD]F}|jd}|r!|j|j|7|jHd}||}t!|gd|jdd }fd!||}t"fd"t$d#t&fd$}|||}d%t$ffd& d"t$ffd' }||}fd(|}|j d)*D]}|j)t+|j-d+d,j-d-d.}t/|}t1j2}t5}d|_|j9|}|j-d/d0} t;||}||d|| |d2S#t<$r!}t?d1t+|i}Yd}~.d}~wwxYw#t<$r)}t?d3t+|td4||d}~wwxYw)5a4 Extract structured content, media, and links from website HTML. How it works: 1. Parses the HTML content using BeautifulSoup. 2. Extracts internal/external links and media (images, videos, audios). 3. Cleans the content by removing unwanted tags and attributes. 4. Converts cleaned HTML to Markdown. 5. Collects metadata and returns the extracted information. Args: url (str): The website URL. html (str): The HTML content of the website. word_count_threshold (int): Minimum word count for content inclusion. Defaults to MIN_WORD_THRESHOLD. css_selector (Optional[str]): CSS selector to extract specific content. Defaults to None. Returns: Dict[str, Any]: Extracted content including Markdown, cleaned HTML, media, links, and metadata. Nrz;Invalid CSS selector , No elements found for CSS selector: divinternalexternalaT)hrefr+rrrr)r+rr(scriptstylelinkmetanoscriptimgimagesvideosaudiosr5srcaltimage)r8r9r2videor6audior7c\|jdD]}|j|_|S)Npre)rget_textstring)nodechilds rEreplace_pre_tags_with_textz:get_content_of_website..replace_pre_tags_with_texts/u- 0$~~/  0Krrr+r$rrrrrrrrrrrr r rrrrrrF)rcD|jD]}t|tjs||t |j dj }t |jdk(r|j dr||ks|j|S)NTrr)contents isinstancerrrQr?rR decompose)rAword_count_thresholdrB word_count(remove_empty_and_low_word_count_elementss rErLzHget_content_of_website..remove_empty_and_low_word_count_elementss *eW[[1<3"%U^^$^%?%E%E%G!HJENN+q0d9S#&::) *KrbodyrJcDg}|jdD]r}|js|jjs+t|jjj }||ksb|j |t|D]}|j |S)NT)rr@rrQrRrUrI)rMrJtags_to_removerrKs rEremove_small_text_tagsz6get_content_of_website..remove_small_text_tags s N}}T* 3::#**"2"2"4!$SZZ%5%5%7%=%=%?!@J!$88&--c2 3&   Krrct|tr|j S|jsyt fd|jDS)NTc3.K|] }|ywrr).0rBis_empty_or_whitespaces rE zIget_content_of_website..is_empty_or_whitespace..*sO-e4Os)rHr rrGall)rrTs rErTz6get_content_of_website..is_empty_or_whitespace$s:#/99;&<<O#,,OO Orcd}|rEd}|jdDcgc] }|s |}}|D]}|jd}|rE|Scc}w)NTF)rrI)rMchangesr empty_tagsrTs rEremove_empty_tagsz1get_content_of_website..remove_empty_tags,snG$(==#6:PQT:UC &#CMMO"G# Ks A A c0|jD]}t|tjs|t |jdk(s?|jdj |j k(sf|jd}|j ||SNr r)rGrHrrrQnamer)rArB child_contentflatten_nested_elementss rEr_z7get_content_of_website..flatten_nested_elements@s} :eW[[1+E2ENN+q0!NN1-22ejj@).q(9 **=9 :Krc"t|tSr)rHrrs rErz(get_content_of_website..Qs D'9Rr)r@ rHrJrP ``````Error extracting metadata:markdown cleaned_htmlrEmedialinksmetadatazError processing HTML content:zInvalid CSS selector: ) rrMselectrnew_tagrUrrRrr?rIr]attrsrr new_stringr$rrrqextractrpr:rr HTML2Textr ignore_linkshandleextract_metadatarr) rr$rJ css_selectorkwargsrrMselected_elementsdiv_tagelrir*r+url_baserrhr3r;r<alt_textrCrPrZcommentrghrfr1rr_rTrLs @@@rEget_content_of_websiter}ysc.dVT=1yy  $ L 9 $-QR^Q_`ll5)G' #r" #DR0s. OAV9Dyy~a(Hv&84+?j!(($ )MNj!(($ )MN  O==!PQ C MMO ==? Cxx5    r:==' C (O " "swwu~wO   ]]7+ E (O " " %(51A7S   ]]7+ E (O " " %(51A7S   ==' Cwwu~H  !:;    *$/#  .jje43 : 8>RS4F  -0 .&d,@A P P C "!& 't,}},R}S G OO  4y((6>>tSI %\2     !  88L)##Iu5 #D$/D!(     .A 7D  V .A7%(>|n&MNTUUVsNP EP I8P  OP  P&P=P PP P<$P77P<rr$rJrtc |syt|d}|j}jdtjdgxsgD](}|j |D]}|j *|rJ|j |} | st d||jd}| D]}|j|ggdgggdfd fd d tjd tffd |jdfdtDD cgc]} | | c} d<|fd|}tjd} D];} | jdd} | j!| r| j#d| | d<=t'|j)ddj)dd}t+|}t-}d|_|j1|}|j)dd} t3||}||d|dScc} w#t$$r }Yd}~d}~wwxYw#t$$r!}t5dt'|i}Yd}~Id}~wwxYw)a Extracts and cleans content from website HTML, optimizing for useful media and contextual information. Parses the provided HTML to extract internal and external links, filters and scores images for usefulness, gathers contextual descriptions for media, removes unwanted or low-value elements, and converts the cleaned HTML to Markdown. Also extracts metadata and returns all structured content in a dictionary. Args: url: The URL of the website being processed. html: The raw HTML content to extract from. word_count_threshold: Minimum word count for elements to be retained. css_selector: Optional CSS selector to restrict extraction to specific elements. Returns: A dictionary containing Markdown content, cleaned HTML, extraction success status, media and link lists, and metadata. Raises: InvalidCSSSelectorError: If a provided CSS selector does not match any elements. Nr$image_description_min_word_threshold excluded_tagsz:Invalid CSS selector, No elements found for CSS selector: r&r'r4c|}|rB|j}|r1|jdd}t|jk\r|S|rBy)NrPT) separatorr)parentr?rQrR)r current_tag text_contentrs rE$find_closest_parent_with_useful_textzNget_content_of_website_optimized..find_closest_parent_with_useful_textsU %,,K*33c3N |))+,0TT''rc0d}d}|||j|jjdgsy|||||}|tkry|jddjddj |jd d||d d S) a0 Processes an HTML image element to determine its relevance and extract metadata. Evaluates an image's visibility, context, and usefulness based on its attributes and parent elements. If the image passes validation and exceeds a usefulness score threshold, returns a dictionary with its source, alt text, contextual description, score, and type. Otherwise, returns None. Args: img: The BeautifulSoup image tag to process. url: The base URL of the page containing the image. index: The index of the image in the list of images on the page. total_images: The total number of images on the page. Returns: A dictionary with image metadata if the image is considered useful, or None otherwise. c |jdd}|jdd}gdddg}td|v|tfd||jd dg|D |j|vgS) Nr/rKr8)buttoniconlogorinputz display:nonec34K|]}D]}||v ywrr)rSrrclasses_to_checks rErUzbget_content_of_website_optimized..process_image..is_valid_image..s2!1S sr9)rrVanyr])r3rparent_classesr/r8 tags_to_checkrs @rEis_valid_imagezOget_content_of_website_optimized..process_image..is_valid_imagesGGGR(E''%$C9 %w/M"%/$');#Mn#M KK}4   rc<d}d}|jd}||\}}|jd} || \} } d} tjj|jdddj j d d} |r|d k(r |d kDr| dz } |d vr |d kDr| dz } | r| d k(r | d kDr| dz } | d vr | d kDr| dz } | dkDr| dz } |jddk7r| dz } t fddDr| dz } ||z dkr| dz } | S)a< Scores an HTML image element for usefulness based on size, format, attributes, and position. The function evaluates an image's dimensions, file format, alt text, and its position among all images on the page to assign a usefulness score. Higher scores indicate images that are likely more relevant or informative for content extraction or summarization. Args: img: The HTML image element to score. base_url: The base URL used to resolve relative image sources. index: The position of the image in the list of images on the page (zero-based). images_count: The total number of images on the page. Returns: An integer usefulness score for the image. c|rKtjd|}|r3t|jd}|jdxsd}||fSy)Nz (\d+)(\D*)r rpxr)r;r?rqr) dimensionr?numberunits rEparse_dimensionztget_content_of_website_optimized..process_image..score_image_for_usefulness..parse_dimensionsNHH]I>E!$U[[^!4!KKN2d &t|+!rct||jd} tj|}|jdk(r|j jddSt d|y#t$rYywxYw)a Fetches the file size of an image by sending a HEAD request to its URL. Args: img: A BeautifulSoup tag representing the image element. base_url: The base URL to resolve relative image sources. Returns: The value of the "Content-Length" header as a string if available, otherwise None. r8rzContent-LengthNz!Failed to retrieve file size for )rrrequestshead status_codeheadersrr )r3base_urlimg_urlrs rEfetch_image_file_sizezzget_content_of_website_optimized..process_image..score_image_for_usefulness..fetch_image_file_sizesx"(CGGEN; '}}W5H++s2'//334DdKK A'KL#$  s?A,A,, A87A8heightr3rr8rKr rr)rvhvminvmax'r9c3(K|] }|k( ywrr)rSr image_formats rErUznget_content_of_website_optimized..process_image..score_image_for_usefulness..$sOf<6)Os)jpgpngwebp?)rrr9splitextrLrr)r3rindex images_countrr image_height height_value height_unit image_width width_value width_unit image_sizescorers @rEscore_image_for_usefulnessz[get_content_of_website_optimized..process_image..score_image_for_usefulnesssV " .778,L(7 (E %L+'''*K&5k&B #KJ77++CGGE2,>?BHHJL'--c2LE$&<#+=QJE"==,QSBSQJE%+*;QJE!<<rAQQJEE! wwu~# O8NOO |#c) LrclassNr8rKrrr9r:)r8r9descrr2)rrrr:r)r3rr total_imagesrrrrs rE process_imagez7get_content_of_website_optimized..process_images  &N `c3::szz~~gr/JK*3ULI ) )775"%--eS9??A775"%8=   rrrLc 0 t|tr"t|tr|jy|jdvr|j yd}|jdk(r|j drn|d}jdd}||jd}|jdr||vr d j|n d j|d }n|jd k(ry |jd vrʼn |jdj|j d|j d|j |d|jd}|D]W} |jdj|j d|j d|j |dYy |jdk7rj|jdvrF j ddr"|j|jn(|jn|jd k7ri|_t|j D]M}t|tr0t|ts t#|j%dkDs@d }C|sLd }O|s/t#|jd j}|k\}|s|j |S#t&$r} t)dt+| Yd} ~ yd} ~ wwxYw)NFr-r*r+rrr,rr)r(Tr3)r;r<rr8r9)r8r9r2 descriptionsourcer>rDrrrFzError processing element:)rHr rror]rIrrRr?rrUrrunwraprmlistchildrenrQrrrrp)r keep_elementr+ry link_data source_tags source_tagrBrKrrrurirhprocess_elementrrJs rErz9get_content_of_website_optimized..process_element7se '?3gw/#GOO%||NN!!!# L||s"{w{{6':v99S>!,%)3C73C3C3EF ??6*xt/C*%,,Y7*%,,Y7# &!33a()00*w{{51*w{{51 ' 'KG'T /g..x8 "- JW\\N!,-44#->>%#8#.7;;u#5$+LL+O ',   ||u$<<$.zz+u5,,,-=W-=-=-?@&(\\U*$&GMg../ ,e_5j7?5;;=)A-'+ &u-'+  ,  !1!1!1!=!C!C!EF )-AA !!!#   -s1v 6 s>1K-K-B K-6CK- C K- K-#A K-- L6LLr3c 3LK|]\}}||tywrrQ)rSr+r3imgsrrs rErUz3get_content_of_website_optimized..s) 6.flatten_nested_elementss dO ,K  ! #4==+W[[9 a %%2*4==+;< rrr;compiler?rrrpr:rrrqrrrsr)rr$rJrtrurrMrrxrvrbase64_patternr3r8rrgr|rfr1rrr_rrrirhrrs` ` ` @@@@@@@@rE get_content_of_website_optimizedrws0  } -D 99D+1::.0T,(zz/2.4"++c" B LLN  KK 5 )L\N[ ||E"# B KKO  ,ER2 6E ~ @f!4!4fffR == D @I$     E(OD  #4 (DZZ BCN ''%$C##C(+//C8E t9$$VT2::4EL .LAANxx %H 51Hd+ $  ]:     *CF3s0* H%+8H* I* H=8H= I* I%%I*c i}|s|iS| tjj|}|j d}|s|S|d}|j d}|r|dnd}|s!|j d}| |j nd}|s-|j dxs|j d}|r|dnd}|r|jnd|d<|j d }|r|djnd|d <|j d }|r|djnd|d <|j d } | r| djnd|d<|j d} | D]M} | jddj} | jddj} | sF| sI| || <O|j d}|D]M} | jddj} | jddj} | sF| sI| || <O|j d}|D]M} | jddj} | jddj} | sF| sI| || <O|S#t$ricYSwxYw)zG Extract metadata from HTML using lxml for better performance. N//headrz.//title/text()z.//titlez%//meta[@property='og:title']/@contentz&//meta[@name='twitter:title']/@contentrz%.//meta[@name="description"]/@contentrz".//meta[@name="keywords"]/@contentkeywordsz .//meta[@name="author"]/@contentauthorz&.//meta[starts-with(@property, "og:")]propertyrKrz'.//meta[starts-with(@name, "twitter:")]r]z+.//meta[starts-with(@property, "article:")]) lxmlr$document_fromstringrxpathfindrrr)r$r]rjrrtitle_eltitle_candidatesrrrog_tagsr property_namer twitter_tags article_tagss rEextract_metadata_using_lxmlrsH CK  { ))//5C 99X D  7D JJ( )EE!H4E 88J'!)!5 4  II= > @ II> ? (8 #T*/ DHW**DEK8Ck!n224H]zz>?H2:8A;,,.HZZZ: ;F.4*$HXjjABG. B/557 '')R(..0 W&-H] # .::GHL.+113 '')R(..0 W&-H] # .::KLL. B/557 '')R(..0 W&-H] # . OO I sI44 JJci}|s|siS|s t|d}|j}|s|S|jd}|r&|jr|jj nd|d<|jdddi}|r |j dd j nd|d<|jddd i}|r |j dd j nd|d <|jddd i}|r |j dd j nd|d <|j dd tjd i}|D]M} | j d d j } | j dd j } | sF| sI| || <O|j ddtjdi} | D]M} | j dd j } | j dd j } | sF| sI| || <O|j dd tjdi} | D]M} | j d d j } | j dd j } | sF| sI| || <O|S)aR Extract optimized content, media, and links from website HTML. How it works: 1. Similar to `get_content_of_website`, but optimized for performance. 2. Filters and scores images for usefulness. 3. Extracts contextual descriptions for media files. 4. Handles excluded tags and CSS selectors. 5. Cleans HTML and converts it to Markdown. Args: url (str): The website URL. html (str): The HTML content of the website. word_count_threshold (int): Minimum word count for content inclusion. Defaults to MIN_WORD_THRESHOLD. css_selector (Optional[str]): CSS selector to extract specific content. Defaults to None. **kwargs: Additional options for customization. Returns: Dict[str, Any]: Extracted content including Markdown, cleaned HTML, media, links, and metadata. rrNr1r]r)rmrrKrrrz^og:z ^twitter:z ^article:) rrrr@rrrr;r)r$rrjr title_tagdescription_tag keywords_tag author_tagrrrrrrs rErsrs/s,H  T6* 99D  '"I$-)2B2B   W iiv}.EiFO6EIr*0024 ] 99VFJ+?9@L3? B'--/T Z 6&();RVHXmmF:rzz'7J*KmLG. B/557 '')R(..0 W&-H] # .== <8P/Q=RL.+113 '')R(..0 W&-H] # .== BJJ|)r;findallrset)r@rs rEextract_xml_tagsrs" ::j& )D D ?rci}|D]Z}d|d|d}tj||tj}|r#|jdj ||<Vd||<\|S)a Extract data for specified XML tags from a string. How it works: 1. Searches the string for each tag using regex. 2. Extracts the content within the tags. 3. Returns a dictionary of tag-content pairs. Args: tags (List[str]): The list of XML tags to extract. string (str): The input string containing XML data. Returns: Dict[str, str]: A dictionary with tag names as keys and extracted content as values. r>(.*?)D#1G!%)RJ)/(? $% zz,&./& !#)6KLMH O  %s1v .,**))"&8'&AB UG+FGH 5! "#!( $O#P" G s+A00C05ACC0 C0)C++C0cKddlm} ddlm} ddl} ddl} d| _d||d} |rdd i| d <|j d r| j|d t|D]} | d|d |d gd| d{}|cSy7 #| $rs}tdt|||dz k(r||dz kr3|||zz}td|d| j|d{7nddgdgdgcYd}~cSYd}~d}~wt$r}|d}~wwxYww)a Async version: Perform an API completion request with exponential backoff. How it works: 1. Sends an async completion request to the API. 2. Retries on rate-limit errors with exponential delays (async). 3. Returns the API response or an error after all retries. Args: provider (str): The name of the API provider. prompt_with_variables (str): The input prompt for the completion request. api_token (str): The API token for authentication. json_response (bool): Whether to request a JSON response. Defaults to False. base_url (Optional[str]): The base URL for the API. Defaults to None. base_delay (int): The base delay in seconds. Defaults to 2. max_attempts (int): The maximum number of attempts. Defaults to 3. exponential_factor (int): The exponential factor. Defaults to 2. **kwargs: Additional arguments for the API request. Returns: dict: The API response or an error message after all retries. r) acompletionrNTrrr2rrrrrrrr r r r@r r r) r rrrasynciorrrrsrrprr)rrrrrrrrrurrr rrrrrrs rE aperform_completion_with_backoffr)sVD$1G!%)RJ)/(? $% zz,&./& (#)6KLMH O   %s1v .,**))"&8'&AB UG+FGH#gmmE*** "#!( $O#P+ G saAD !B3A>4B9D >BDAC3CC3&D' D 3 D?DDD c2|stj|dn|}|tt|d}t}|D]}|j d|zdz||}t ||||} tdg|jdjjd} tj| } | D]} d| d <  | S#t$rNt|jdjj\} } | } | r| jdd d g| d Y| SwxYw) a Extract content blocks from website HTML using an AI provider. How it works: 1. Prepares a prompt by sanitizing and escaping HTML. 2. Sends the prompt to an AI provider with optional retries. 3. Parses the response to extract structured blocks or errors. Args: url (str): The website URL. html (str): The HTML content of the website. provider (str): The AI provider for content extraction. Defaults to DEFAULT_PROVIDER. api_token (Optional[str]): The API token for authentication. Defaults to None. base_url (Optional[str]): The base URL for the API. Defaults to None. Returns: List[dict]: A list of extracted content blocks. NURLHTMLrrrblocksrFr@T)rr@rr)rrrrr r:rrchoicesr1rrrrrrU) rr$rrrvariable_valuesrvariablerr%blockrunparseds rEextract_blocksr+zsL*.>q.A.I.I.Q.QR  F# #E"E'N # M  7   Q  ' ' / /   MMdWI(S  M s*AB??ADDc|stjddn|}ddlm}g}|D]G\}}||d}t}|D]} |j d| zdz|| }|j d|d gI|||d } g} | D]Z} td g| jdjjd } tj| } | j | \t| gS#t$rdd gdgdgdg} Y5wxYw)at Extract content blocks from a batch of website HTMLs. How it works: 1. Prepares prompts for each URL and HTML pair. 2. Sends the prompts to the AI provider in a batch request. 3. Parses the responses to extract structured blocks or errors. Args: batch_data (List[Tuple[str, str]]): A list of (URL, HTML) pairs. provider (str): The AI provider for content extraction. Defaults to "groq/llama3-70b-8192". api_token (Optional[str]): The API token for authentication. Defaults to None. Returns: List[dict]: A list of extracted content blocks from all batch items. GROQ_API_KEYNr)batch_completionr!rrrrr)rrrr%r@zZError extracting blocks from the HTML content. Choose another provider/model or try again.z4What went wrong during the block extraction process?)rrr questions)rrr r.r r:rUrr&r1rrrrrT) batch_datarrr.rr_htmlr'rr( responses all_blocksrr%s rEextract_blocks_batchr4sU$8A .$/iI(H N U  !6' H$9$A$Ah$oh&?% !  &5JKLM N!x(PTUIJ" %xj(2B2B12E2M2M2U2UVFZZ'F &!)", z2  $It O"  F sAC''C>=C>c(g}g}d}|D]d}t|jdz}||z|kr|j|||z }>|r |jdj||g}|}f|r |jdj||S)a Merges small chunks into larger ones based on the total token threshold. :param chunks: List of text chunks to be merged based on token count. :param token_threshold: Max number of tokens for each merged chunk. :return: List of merged text chunks. rg?ra)rQrRrUrY)r{token_thresholdmerged_sections current_chunktotal_token_so_farr~chunk_token_counts rE%merge_chunks_based_on_token_thresholdr;sOM 3   $   1 1O C   ' "3 3 &&v{{='AB"GM!2  3v{{=9: rsectionsrrc g}|jdr=|D]6}|jt|||||tjd8|St 5}|Dcgc]}|j t|||||}}t|D]!} |j| j# ddd|Scc}w#1swY|SxYw)a Process sections of HTML content sequentially or in parallel. How it works: 1. Sequentially processes sections with delays for "groq/" providers. 2. Uses ThreadPoolExecutor for parallel processing with other providers. 3. Extracts content blocks for each section. Args: url (str): The website URL. sections (List[str]): The list of HTML sections to process. provider (str): The AI provider for content extraction. api_token (str): The API token for authentication. base_url (Optional[str]): The base URL for the API. Defaults to None. Returns: List[dict]: The list of extracted content blocks from all sections. zgroq/r$rN) rrSr+rrrsubmitrr) rr<rrrextracted_contentsectionexecutorfuturesfutures rEprocess_sectionsrDs,7# G  $ $sGXy8T  JJsO  "  ! :X ( "C(IPX G 'w/ :!((9 : :  : sC "B>1C>CC c0g}|j}|rrd}|rZ|jd||dz|d|kr:||jddzz }|r!|jd||dz|d|kr:|j||rrdj |S)a3 Wrap text to fit within a specified width for rendering. How it works: 1. Splits the text into words. 2. Constructs lines that fit within the maximum width using the provided font. 3. Returns the wrapped text as a single string. Args: draw (ImageDraw.Draw): The drawing context for measuring text size. text (str): The text to wrap. font (ImageFont.FreeTypeFont): The font to use for measuring text size. max_width (int): The maximum width for each line. Returns: str: The wrapped text. rK)rrr)fontrrPrH)rRtextbboxpoprUrY)drawrrF max_widthlineswordsr_s rE wrap_textrMCs( E JJLE  dmmFD58O$mGJiW EIIaL3& &D dmmFD58O$mGJiW  T  99U rc:t|d}|jS)aO Prettify an HTML string using BeautifulSoup. How it works: 1. Parses the HTML string with BeautifulSoup. 2. Formats the HTML with proper indentation. 3. Returns the prettified HTML string. Args: html_string (str): The HTML string to format. Returns: str: The prettified HTML string. z lxml.parser)rr) html_stringrs rE format_htmlrPcs m 4D ==?rcBd}d}g}|jddjddjd}|D]}|js|jdr|d z}|j ||z|zB|jdr)|j d r|j ||z|z||jdr|j ||z|z|d z }|j}|s|j ||z|zdj |S) z A fast HTML formatter that uses string operations instead of parsing. Args: html_string (str): The HTML string to format Returns: str: The formatted HTML string rrJrz> rz )r:rRrrrUr=rY)rOindent indent_str formattedpartspartrs rEfast_format_htmlrWws$FJI   U + 3 3C ? E Ed KE@zz|  ??4 aKF   Z&047 8__S !dmmD&9   Z&047 8__S !   Z&047 8 aKFjjlG  f!4w!>?-@0 99Y rcddlm}m}||}|jr |jst d||jj dvrt d||j}|||S)*Normalize URLs to ensure consistent formatr)rr,Invalid base URL format: )rhttps) urllib.parserr,rrrrLr)r+rrr, parsed_base cleaned_hrefs rE normalize_urlr_sz.8$K   [%7%74XJ?@@  "*;;4XJ?@@:: refgclidfbclidref_srcutm_term utm_medium utm_source utm_content utm_campaignc |dS)Nrr)kvs rErznormalize_url.. s r!urrdoseqrK)rrr,rrrr:rLr9r=rstripqueryr.sortr/fragmentr-params)r+rr`rarbrcrdrefull_url parsed_fullr]rrr9rykvr|default_trackingrr{ normalizeds rEr_r_sD x.H/W4x( x(    & (   +"4"4 4 ''-'' :qAHh F]] "F ;;D }}Sdck{{3 LLE -6uPT-UVTQ1779a.VV   ! 8I$J1QWWY$JJ )/MA1. s" Ie4::< " "5 ) I),rrg)rhriz./)rR IndexErrorrrrrlstrip)r+r base_partsprotocolrspecial_protocolss` rEnormalize_url_tmpr sA^^C( a=A U I7H IIzz| sD6"" tD6"" s2fXdV,, ??2 3{{4 2fXQtf-- ::<5 A4XJ?@@As C C"cddlm} ||}t|}g}g}t }|j dD]}|j ddj} | r| jdr7t| |} | r| |vrJ|j| |jxsdjdd } | | d } t| |r|j| |j| ||dS#t$rggdcYSwxYw) a8 Fast link extraction for prefetch mode. Only extracts tags - no media, no cleaning, no heavy processing. Args: html: Raw HTML string base_url: Base URL for resolving relative links Returns: {"internal": [{"href": "...", "text": "..."}], "external": [...]} r)rr'z //a[@href]r+rK)rrrrNrr,) lxml.htmlrrget_base_domainrrrrrraddris_external_urlrU) r$rrr] base_domainr(r)seenr*r+rrrs rEquick_extract_linksr s.0!$'"(+K%'H%'HUD YY| $'uuVR &&(t'NO 2$A Z4/   &B--/5'6 :{ 3 OOI & OOI &''*!h 77; 0B//0sC55DDc` t|jj}|sy|jdd}t j dd|}|jd}t |dkDr|ddvrdj|d d Sdj|dd S#t$rYywxYw)  Extract the base domain from a given URL, handling common edge cases. How it works: 1. Parses the URL to extract the domain. 2. Removes the port number and 'www' prefix. 3. Handles special domains (e.g., 'co.uk') to extract the correct base. Args: url (str): The URL to extract the base domain from. Returns: str: The extracted base domain or an empty string if parsing fails. rKrrz^www\.rrr7> acadaeafagcocomedugovrqmilnetorgN) r,rrLrRr;rrQrYr)rrrUs rErr s!#%%++-c"1% 2v. S! u:>eBi,  88E"#J' 'xxbc ## s%B!A$B! B!! B-,B-rc>hd}tfd|Dry t}|jsy|jjj dd}|jj dd}|j | S#t $rYywxYw)rrc3\K|]#}jj|%ywrr)rSrrs rErUz"is_external_url.. s" 6399; ! !! $ 6rTFzwww.rK)rr,rrLr:r=r)rrspecialr url_domainbases` rErr sKG 6g 66 #}}]]((*2262>   "**626&&t,,, sBAB BBr^chd}hd}|Dcgc]M}t|dkDr=||vr9||vr5|jds$|jds|jds|Oc}Scc}w)u Clean a list of tokens by removing noise, stop words, and short tokens. How it works: 1. Defines a set of noise words and stop words. 2. Filters tokens based on length and exclusion criteria. 3. Excludes tokens starting with certain symbols (e.g., "↑", "▲"). Args: tokens (list[str]): The list of tokens to clean. Returns: list[str]: The cleaned list of tokens. >⬆️r*anatbyinofontoupccpthe↑▲>n'tcan'twon'tmustn'tcouldn'twouldn't shouldn'tr*r+amrasrberdoherisitmemynorrorsorruswerVandrarebutcandidfewforhadhasherhimhishowitsmaynornotoffouroutsherwaswhowhyyetyoubeenbothdoesdowneachfromhavehersintominemoremostmustnearnoneoursoverpastsomesuchthatthemtheythisuponwerewhatwhenwhomwillwithyouraboutaboveafteralongamongbeingbelowcoulddoingmightothershallsincetheirthesethoseunderuntilwherewhichwhosewouldyoursacrossaroundbeforebehindbesidebeyondcannotduringexcepthavinginsideitselfmyselfshouldtheirstowardunlesswithinagainstbecausebeneathbetweenherselfhimselfoutsidethroughalthoughyourself ourselves themselves underneathrrru⬆)rQr)r^noise STOP_WORDStokens rE clean_tokensrO) sy" E"tJr   u:>    #  '  '  '    sAA"c.tfd}|S)a Decorator to profile a function's execution time and performance. How it works: 1. Records the start time before executing the function. 2. Profiles the function's execution using `cProfile`. 3. Prints the elapsed time and profiling statistics. Args: func (Callable): The function to decorate. Returns: Callable: The decorated function with profiling and timing enabled. cltj}tj}|j |g|i|}|j tj|z }t d|ddtj|}|jd|jd|S)Nz![PROFILER] Scraping completed in z.2fz seconds cumulative) r perf_countercProfileProfileenabledisablerpstatsStats sort_stats print_stats) rBargsru start_timeprofilerr elapsed_timestatsfuncs rEwrapperz!profile_and_time..wrapper s&&( ##%d,T,V, ((*Z7  1,s1C8LM X& & " rr!)rbrcs` rEprofile_and_timerd s"  4[6 Nrrcdtj|jjS)z"Generate a unique hash for content)xxhashxxh64rr)rs rEgenerate_content_hashrh= s! <<( ) 3 3 55r head_htmlc|sy|j}g}tjd|tj}|r.|j |j dj gd}|D]\}}d|dtj|dd|dtj|d g}|D]J}tj||} | s|j | j dj |syd j|} tj| jjS) a3 Compute a fingerprint of content for cache validation. Focuses on content that typically changes when page updates: - - <meta name="description"> - <meta property="og:title|og:description|og:image|og:updated_time"> - <meta property="article:modified_time"> - <meta name="last-modified"> Uses xxhash for speed, combines multiple signals into a single hash. Args: head_html: The HTML content of the <head> section Returns: A hex string fingerprint, or empty string if no signals found rK��z<title[^>]*>(.*?)r ))r]r)r]z last-modified)rzog:title)rzog:description)rzog:image)rzog:updated_time)rzarticle:modified_timez ]*z=["\']z%["\'][^>]*content=["\']([^"\']*)["\']z*]*content=["\']([^"\']*)["\'][^>]*z["\']r) rLr;rrrUrrr<rYrfrgrr) ri head_lowersignals title_match meta_tags attr_type attr_valuepatternsrDr?combineds rEcompute_head_fingerprintrsC s3& "JG))8*biiPK{((+1134I"+  :)FBIIj,A*BBg h9)F299U_K`Jaaf g   GIIgz2Eu{{1~3356    xx H <<) * 4 4 66r base_pathcddddddd}i}|jD]A\}}tjj||}tj|d|||<C|S) z.Create content directories if they don't exist html_contentrgmarkdown_contentr? screenshots)r$cleanedrf extractedrx screenshotTr)itemsrr9rYr)rtdirs content_pathsrdirnamer9s rEensure_content_dirsr sq!&($#  DM " Www||Iw/ D4(! c" rctjdk(r(tjtjyy)a Configure the Windows event loop to use ProactorEventLoop. This resolves the NotImplementedError that occurs on Windows when using asyncio subprocesses. This function should only be called on Windows systems and before any async operations. On non-Windows systems, this function does nothing. Example: ```python from crawl4ai.async_configs import configure_windows_event_loop # Call this before any async operations if you're on Windows configure_windows_event_loop() ``` rtN)rrrset_event_loop_policyWindowsProactorEventLoopPolicyrrrEconfigure_windows_event_loopr s/ I%%%g&L&L&NO&r context_linescddl}ddl}ddl}|j|d}|d}|j}|j }|j } td||z } ||zdz} g}t| | D]J} |j|| } | s| j} | |k(rdnd}|j| dd|d| Ld j|} |jj|}||| |d S#t$r|}YwxYw) a  Extract error context with more reliable line number tracking. Args: exc_info: The exception info from sys.exc_info() context_lines: Number of lines to show before and after the error Returns: dict: Error context information rNrrr rrP4drH)rCline_nofunction code_context) traceback linecacher extract_tbrClinenor]rrrsgetlinerxrUrYr9relpathr)exc_inforrrrtb last_framerCr func_name context_start context_endr+r_pointerrrel_paths rEget_error_contextr s/    hqk *BBJ""HGI7]23MM)A-KM =+ .=  1- ;;=D G|eG  Ab67)1TF!; < =99]+L77??8, $   sC00 C>=C>c2t||kDr|d|dzS|S)N...r)value thresholds rEtruncater s% 5zIZi 5(( Lrc.tjj|}|jD]}t |j D](}t |j |||j |<*|jr3t|j|kDrt |j||_|jst|j|kDst |j||_ tjj|ddS)NunicodeF)encoding pretty_print) rr$ fromstringiterrattribrrrQtailtostring)html_strrroot_elementattrs rE optimize_htmlr s 99   )DIIK ?) OD$,X__T-BI$NHOOD ! O ==S/);$X]]I>HM ==S/);$X]]I>HM ? 99  dYU  KKrcReZdZeddZeddZedefdZdefdZy) HeadPeekrc(Kdddd} tj|4d{}|j||dd{}|j|k7r0t |j}|j||d{}d }|j 23d{}||z }d |vs|j d d d zcdddd{S777V7=627#1d{7swYyxYw#tjtf$rYywxYww) Nz&Mozilla/5.0 (compatible; CrawlBot/1.0)z text/htmlclose)z User-AgentAccept ConnectionrT)rfollow_redirects)rrsr) httpx AsyncClientrrrp aiter_bytesrR HTTPErrorr)rrrclientrrr~s rEfetch_head_sectionzHeadPeekr.fetch_head_section s(C!!   ((9 A AV!'CSW!XX<<3&hll+C%+ZZWZ%EEH#+#7#7#9%u$G!W,}}Z03j@ A A AX F#9 A A A A*  sDC3CC3CC>C?CCCCC C)C C3 C C3DC3CCCCC3C0$C' %C0,C3/D0C33D DDDcxKtj||d{}|r|jddSy7w)Nrrsrr)rrr)rr head_sections rE peek_htmlzHeadPeekr.peek_html s@&99#w9OO &&wx&@ @Ps :8: head_contentcji}d}tj||D]}|jd}tjd|}tjd|}tjd|}|sY|s|s^|r|jdn|jd}|jd||<|S)Nz ]+>rzname=["\'](.*?)["\']zproperty=["\'](.*?)["\']zcontent=["\'](.*?)["\']r )r;finditerrr) rrn meta_patternmeta_tagr name_matchproperty_match content_matchrs rEextract_meta_tagszHeadPeekr.extract_meta_tags s &  L,? 8H..#C#:C@JYY'BCHNII&@#FM*-7j&&q)^=Q=QRS=T!.!4!4Q!7 # 8rctjd|tjtjz}|r|j dSdS)Nz(.*?)r )r;r IGNORECASErr)rrms rE get_titlezHeadPeekr.get_title, s=ii 7r}}WYW`W`G`a '2{  #<idr]r2rrzdata-rz //*[@class]rrr$)rmethod)r# HTMLParserlhtmlrr getparentremoverrrrrrHrQrrrrrfrgitertextrr intdigestrr)rvtext_thresholdattr_value_thresholdmax_sizertree head_elementsrrOrelementsrattribs_to_keepattributes_hates_truncaterrrxrclsr|txtsigrrs rEpreprocess_html_for_schemar0 s>BY!!$$O V< 8, ! .D~~+ ''- .   " 8CzzBse*-H# 8$7$$&2%G%%'..w7 8 8yy{ MG w  "* GO )+ %w~~2245 c/1V5F5Fw5ONN&&v.#<<W^^TZE[A\_sAs-4^^F-CDYEY-Z]b-bGNN6*  c||GLL$6$6$8 9N J&||113O^DuL ||GLL$6$6$8 9N J&||113O^DuL ? MD#%tzz-01 !B\\^F~&&/C A{{}   663 .Cd{v1 b! S ' !xyH v; !)8$u, , Y*-l*;h*F|IX&LXYsLAL9AL9,B+L9L94BL9=&L9%DL97L99 M!MM!M!c ddl}ddlm}ddlm}m}ddl}ddl}ddl }d|jd<|jgd}|jd |jd g}|jgd } |jgd } |jd |jd } ||| dddy#t $r t dwxYw)zm Start virtual display server in Google Colab. Raises error if not running in Colab environment. rN)r)IFramedisplayz6This function must be run in Google Colab environment.:99DISPLAY)Xvfbrz-screen0 1280x720x24rfluxbox) x11vncz-displayrz-nopwz-foreverz-sharedz-rfbport5900z-quiet)z/opt/novnc/utils/websockify/run6080zlocalhost:5900z--webz /opt/novncz#google.colab.kernel.proxyPort(6080)z'/vnc.html?autoconnect=true&resize=scalerpi)r3r) google.colabrIPython.displayrr ImportErrorrrrrenvironPopenreval_js) googlerrrrrrxvfbrrnovncrs rEstart_colab_display_serverr s U'3 !BJJy :  J KDDJJqMj {+GZ  <=F J  5 6EDJJqM ..> ?C FcUAB$WZ [\; USTTUs CCcZddlm}|}td|jdddy)z8 Alternative setup using IPython magic commands r) get_ipythonu7🚀 Setting up Crawl4AI environment in Google Colab...bashrKux set -e echo "📦 Installing system dependencies..." apt-get update -y apt-get install -y xvfb x11vnc fluxbox websockify git echo "📥 Setting up virtual display..." git clone https://github.com/novnc/noVNC /opt/novnc git clone https://github.com/novnc/websockify /opt/novnc/utils/websockify pip install -q nest_asyncio google-colab echo "✅ Setup complete!" N)IPythonrrrun_cell_magic)ripythons rEsetup_colab_environmentr s0$mG CD 62 ( r page_titleheadlines_textmeta_descriptioncpt|xsd|xsdddd ddlm}||}|jj d<t fddDd <|xsdd zd zd zd zj }td |j Dd<S#t$rYSwxYw)a Extract page context for link scoring - called ONCE per page for performance. Parser-agnostic function that takes pre-extracted data. Args: page_title: Title of the page headlines_text: Combined text from h1, h2, h3 elements meta_description: Meta description content base_url: Base URL of the page Returns: Dictionary containing page context data for fast link scoring rKF)terms headlinesrr is_docs_siter)r,rc3,K|] }|dv yw)rNr)rS indicatorcontexts rErUz'extract_page_context..# s%&d(1'0783D&D&ds)zdocs.zapi.z developer.z reference.r rPr rc3xK|]2}t|jddkDr|jd4ywz .,!?;:"()[]{}rN)rQrrSr*s rErUz'extract_page_context..) s9F!% #DJJ$? @1 D $zz/:Fs8:r )rr\r,rrLrrRr)rrrrr,rall_textrs @rEextract_page_contextr s#)r,2 G )(#"MM//1#&&d5b&d#d %2,w{/CCcIGTfLggnnpF)1)9FF N   N  sB B(( B54B5 link_text title_attr class_attrrel_attr page_contextcd} |r!t|jdkDr|dz }|xsdjtfddDr|dz }tfdd Dr|dz}|xsdjtfd d Dr|dz }tfd d Dr|dz}|jtfddDr|dz }ntfddDr|dz }tfddDr|dz}|j ddz }|dkr|dz }n |dkDr|dz}|j dr|dz }rkj}t|dkDr|dz }t|j } | dk\r|dz }| dk\r|dz }gd} |j| vr|dz}|jdrZrXtdj D} | r6t| |dz} | dkDr | tt| dz } || dzz }|jd d!rrtfd"d#Dr|dz }tdt|d$S#t$rd}Y#wxYw)%a Ultra-fast link quality scoring using only provided data (no DOM access needed). Parser-agnostic function. Args: link_text: Text content of the link url: Link URL title_attr: Title attribute of the link class_attr: Class attribute of the link rel_attr: Rel attribute of the link page_context: Pre-computed page context from extract_page_context() Returns: Quality score (0.0 - 10.0), higher is better rNrOrKc3&K|]}|v ywrr)rS nav_class class_strs rErUz1calculate_link_intrinsic_score..T sg)yI%g)navmenuprimarymain importantg?c3&K|]}|v ywrr)rS bad_classrs rErUz1calculate_link_intrinsic_score..W se)yI%er)rsponsorrpromobannerc3&K|]}|v ywrr)rSgood_relrel_strs rErUz1calculate_link_intrinsic_score..\ s\xx7"\r) canonicalnextprevchapterc3&K|]}|v ywrr)rSbad_relr,s rErUz1calculate_link_intrinsic_score..^ sRgw'!Rr)nofollow sponsoredugcrc3&K|]}|v ywrr)rS good_path url_lowers rErUz1calculate_link_intrinsic_score..e s)yI%r)z/docs/z/api/z/guide/z /tutorial/z /reference/z/manual/g@c3&K|]}|v ywrr)rS medium_pathr8s rErUz1calculate_link_intrinsic_score..g sik )ir)z/blog/z /article/z/post/z/news/c3&K|]}|v ywrr)rSbad_pathr8s rErUz1calculate_link_intrinsic_score..k szx9$zr)z/admin/z/login/z/cart/z /checkout/z/track/z/click/rrrirI)z click herez read morez more infor0herer c3K|]@}t|jddkDr!|jdjBywr)rQrrLrs rErUz1calculate_link_intrinsic_score.. sBD#!$**_"=>B"ZZ8>>@DsAAr r Fc3BK|]}|jvywr)rL)rS doc_keywordrs rErUz1calculate_link_intrinsic_score.. s'!h#."- 0A!A!hs)api referenceguidetutorialexample$@) rQrrLrrxrrRrrrhrrr)rrrrrrr url_depth text_cleanrK generic_texts link_wordsrIrelevance_ratiorr,r8s` @@@rEcalculate_link_intrinsic_scorerN4 s. ET #j..01A5 SLE%2,,. g7fg g SLE e7de e SLE>r((* \3[\ \ SLE R1QR R SLEIIK  7~  SLE i=hi i SLE z5yz z SLEIIcNQ& > SLE ] SLE >>* % SLE "*J:" Z--/0JQ Q UM!]2    G $D'0'8DDJj<+@@AQ;&-C OR0H&HO_s22E   NE 2S!h2f!hh  sCt$ %% sII-- I;:I;intrinsic_scorecontextual_scorescore_links_enabledquery_providedc|sy||nd}||nd}|r|tdt|dSt|dzd}|dz|dzz}tdt|dS)a Calculate combined total score from intrinsic and contextual scores with smart fallbacks. Args: intrinsic_score: Quality score based on URL structure, text, and context (0-10) contextual_score: BM25 relevance score based on query and head content (0-1 typically) score_links_enabled: Whether link scoring is enabled query_provided: Whether a query was provided for contextual scoring Returns: Combined total score (0-10 scale) Scoring Logic: - No scoring: return 5.0 (neutral score) - Only intrinsic: return normalized intrinsic score - Only contextual: return contextual score scaled to 10 - Both: weighted combination (70% intrinsic, 30% contextual scaled) g@rNrHgffffff?r)rrrh)rOrPrQrR intrinsic contextualcontextual_scaledtotals rEcalculate_total_scorerX s2 $3#>CI%5%A!sJ -53It,--J-t4_!2S!8 9E sCt$ %%rtexts llm_config model_name batch_sizecKddl}|s|jgS|ddlm}|j dd}|j d|j d}|||j d|j d d }|r||d<|r d |vrd ||d <|di|d{} g} | j D]} | j | d |j| S ddlm} ttds it_ |tjvr| |tj|<tj|} | j||dd} | S7#t$r tdwxYww)ah Compute embeddings for a list of texts using specified model. Args: texts: List of texts to embed llm_config: Optional LLM configuration for API-based embeddings model_name: Model name (used when llm_config is None) batch_size: Batch size for processing Returns: numpy array of embeddings rN) aembeddingrztext-embedding-3-smallrapi_baserr)rrrzopenai/r embedding)SentenceTransformerzsentence-transformers is required for local embeddings. Install it with: pip install 'crawl4ai[transformer]' or pip install sentence-transformers_modelsFT)r\show_progress_barconvert_to_numpyr)numpyr r r^rrrUsentence_transformersrarhasattrget_text_embeddingsrbr)rYrZr[r\npr^embedding_modelr_rur embeddingsitemraencoders rErhrh s$ rxx|&%..5MN>>*jnnZ.HI%!~~k:>>)3LM  !)F:   8 ''89F7O$-f-- MM 1D   d;/ 0 1rxx ##   A*I6*,  ' 088 86I*6U  ' ' 3%--j9^^ !#! $ K. l  s+BE*E=E*E A1E*E''E*cLddl}tjt||||S)z+Synchronous wrapper for get_text_embeddingsrN)rerrunrh)rYrZr[r\ris rEget_text_embeddings_syncrp.s" ;;*5*j*U VVrvec1vec2cddl}|j||}|jj||jj|z}|dk7rt ||z SdS)z/Calculate cosine similarity between two vectorsrNrN)redotlinalgnormfloat)rqrrri dot_product norm_products rEcosine_similarityrz9sW"&&t$K99>>$'"))..*>>L0<0A5|+ ,JsJrc dt||z S)z>Calculate cosine distance (1 - similarity) between two vectorsr )rz)rqrrs rEcosine_distancer|As  t, ,,rcftj}tjdk(ra t j dgdd}|j jd}d}i}|D]}d|vr/t|jdjd |d <6d |vr/t|jdjd |d <id |vr/t|jdjd |d<d|vst|jdjd |d<|jd d|jd dz|jddz|jddz}||zdz }|S|jdz S#|jdz cYSxYw)zDGet truly available memory including inactive pages (cross-platform)rqvm_statT)capture_outputrrHi@z Pages free:rrfreezPages inactive:inactivezPages speculative: speculativezPages purgeable: purgeablerre) psutilvirtual_memoryrrrrostdoutrRrqrxr available)vmrrK page_sizepagesr_total_available_pages available_gbs rEget_true_available_memory_gbrHs    BH$ ,^^YK4PFMM''-EIE K D($' R(8(?(?(D$EE&M&$.(+DJJL,<,C,CC,H(IE*%)T1+.tzz|B/?/F/Fs/K+LE-('4/),TZZ\"-=-D-DS-I)JE+& K &!$ *a() -+, +q)* " 2I='JL  ||w''  ,<<7+ +sCF BFF0ctj}|jdz }t}d||z z|z }t dt d|S)z Get memory usage percentage that accounts for platform differences. Returns: float: Memory usage percentage (0-100) regY@rN)rrrWrrrrhrtotal_gbr used_percents rEget_true_memory_usage_percentrrsT    Bxx7#H/1LH|34x?L sC|, --rcztj}|jdz }t}t }|||fS)z Get comprehensive memory statistics. Returns: Tuple[float, float, float]: (used_percent, available_gb, total_gb) re)rrrWrrrs rEget_memory_statsrs=    Bxx7#H/1L02L x //rhooksc 4i}|jD]Z\}}t|std|dt| t j |}t j|}|||<\|S#ttf$r}td|d|d}~wwxYw)a Convert hook function objects to string representations for Docker API. This utility simplifies the process of using hooks with the Docker API by converting Python function objects into the string format required by the API. Args: hooks: Dictionary mapping hook point names to Python function objects. Functions should be async and follow hook signature requirements. Returns: Dictionary mapping hook point names to string representations of the functions. Example: >>> async def my_hook(page, context, **kwargs): ... await page.set_viewport_size({"width": 1920, "height": 1080}) ... return page >>> >>> hooks_dict = {"on_page_context_created": my_hook} >>> api_hooks = hooks_to_string(hooks_dict) >>> # api_hooks is now ready to use with Docker API Raises: ValueError: If a hook is not callable or source cannot be extracted zHook 'z#' must be a callable function, got z%Cannot extract source code for hook 'zK'. Make sure the function is defined in a file (not interactively). Error: N) r|callablerr2inspect getsourcerMdedentr TypeError)rr hook_name hook_funcrrs rEhooks_to_stringrs4F %  9 "vi[0STXYbTcSdef f &&y1F__V,F &F9  M # 7 {C[[\Z]_  s/A22BBB)g?N)rrON)rCxTF)Fr)FNrrr)zgroq/llama3-70b-8192N)FN)r=)r)dri)NNFF)Nz&sentence-transformers/all-MiniLM-L6-v2 )rconcurrent.futuresrrbs4rrrrr rr$rr;rrrpromptsr r r rconfigrrrrrrsocketrpathlibrtypingrrrrrrrrr\rrrequests.exceptionsr rfrMrUrY functoolsr"rr#rrrurllib.robotparserr%rr& packagingr'rKr(r) itertoolsr* collectionsr+rrerir,r-r.r/r0r1rr2 applies_torArFrprqrwrirrrrr bytearrayr'r)rr0r>rcrlrgrrrrrrrr$r}rrrsrrrrrr+r4r;rDrMrPrWr_rrrrrrrOrdrhrsrrrrrrrrrdictrrNrXndarrayrhrprzr|rrrrrrrErs  ?EE *1GG RRR -  %. ( )) .)"6: T$}T$T$T$ T$ #S !123 T$ sD$ T$r! >: 3->:>:>: >:  >:  #Y >:B882|Z|Z~ i      !"#$%&'()*+,-./0123456789:;<=>?@ABCDE               !" #$                          !"#$%&'()*+,-./012345678       !"#$%&'()*+,-./012345678 ! ! ! !$ cCi% , !!c!J M M M M M  M  M`-*36l23S3j(*-Z2EEE$ <3B%7T{VB!3 _ _ __ _  #s(^ _D SlPf :J M~ Nb(84RV8v>BBGK* **(+*8;* *Z@(+ \+0b bbJ8t 5$$N!H-8c-8S-8T#tDcN?S:S5T-8`000f34DbcbtCybJ,^63636 :7:7:7z34S>(P(6s6p L$7=7=rOYb']V6*S*#*QT*`c*hl*Zp&p& p&p& p&  p&  p& p&h(,(, % -&e_-&uo-&-& -&  -&f"&> S 9SSS S ZZ Sp"&> W 9WWW W ZZ WKBJJKbjjKUK-"**-BJJ-5-'(e'(T.u.$ 0%ue 34 0 ,4X .,4S>,r