/i4ddlZddlZddlZddlZddlZddlmZmZmZm Z ddl Z ddl Z ddl Z ddl mZddlmZddlmZddlmZmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)dd l*m+Z+dd l,m-Z-eZ.d e/fd Z0d e/fdZ1d e2e3e3ffdZ4de3de3de3de3de3f dZ5d ee3effdZ6dee3d e/fdZ7dee3d e/fdZ8de3dedede9fdZ:dZ;de3d ed"e"fd#Z?d"e"fd$Z@d%ZAd&ZBd'ZCejd(d)d*gi+d,ZEeEjd-d.ZFeFjd/d0ZHeFjd1ejd2d3ejd4d5gd4d67ejd8d9e?d@e3dAe?d@ee3dAee<dBee9fdOZNeEjdPejdQdRdSJejd8dTeVejdWd=dXVdYee3dAe z <|end of context|> userT)modelapi_keymessagesstreamchoicesrdeltarI)endflush)rr:print)rCrDrEr?r@responsechunkrIs r/stream_llm_responserXTsFGJFKKLM   4H:=UV[U\]   H /I&q)'266yA A7 A 'r ./ Gs AA!A!cj|siSi}|jd}|D]<} |jdd\}}|jdk(rd}n|jdk(rd}n|jr t|}n|j dd djr t |}n|j d rR|jd rA|dd jdDcgc]#}|js|j%}}n8|j d r'|jdr tj|}|||j<?|Scc}w#tj$rtjd|wxYw#t$rtjd|wxYw)N,=trueTfalseFrGrR[]{}zInvalid JSON object: zInvalid key=value pair: )splitlowerisdigitintreplacefloatr>endswithstripjsonloadsJSONDecodeErrorr; BadParameter ValueError) ctxparamvalueresultpairspairkvxs r/parse_key_valuesrzls  F KK EH H::c1%DAqwwyF"Ag%5q#a&a3r!$,,.E!Hc"qzz#()!B c(:H1aggiQWWYHHc"qzz#J 1 A!"F1779 H$ MI++J,,/DQC-HIIJ H$$'?v%FG G Hs<CF!E7E %F/E!FF!,F  F"F2pathc @|siS t|5}|jdrtj|cdddSt j |cdddS#1swYyxYw#t $r)}tjd|dt|d}~wwxYw)N)z.yamlz.ymlzError loading config file z: ) r)rjr*r+rlload Exceptionr;rostr)r{r.es r/load_config_filers  P $Z 1}}./~~a(  99Q<    P  #=dV2c!fX!NOOPs? A+&A A+A A+A($A+(A++ B4$BBc|syt|S)N)r)r{s r/load_schema_filers  D !!rB browser_cfg crawler_cfgverbosecK|ratjdtjd|jtjd|jt|4d{} |j ||d{}|cdddd{S7377 #t $r&}tj dt|d}~wwxYw#1d{7swYyxYww)Nz%Starting crawler with configurations:zBrowser config: zCrawler config: )r1)rCr1zCrawling failed: )r;r<r4r arunr~ClickExceptionr)rCrrrcrawlerrtrs r/ run_crawlerrs :; %k&6&6&8%9:; %k&6&6&8%9:;k2EEg E"<>"%bggoob11  22 rBprofilesc |s"tjtdddytdddd}|j d d d |j d dd|j dd|j dd|j dd|j dddt |D]u\}}t |d}tj|}|djd}|jt|dz|d|d||dj|wtj|y) z(Display a rich table of browser profileszI[yellow]No profiles found. Create one with the 'create' command.[/yellow]zBrowser Profilesbluetitle border_styleNT bold cyanr show_header header_styler#dim)stylewidthNamecyan)rno_wraprgreenrCreatedyellowBrowsermagentaSizeright)rjustifyr{createdz%Y-%m-%d %H:%Mr\nametype) consolerUr r add_column enumeraterhumanize naturalsizestrftimeadd_rowr capitalize)rtableiprofilesize human_sizers r/display_profiles_tablersH  eg 2I J *;ek lE SQ/ V648 V7+ Yh/ Yi0 V67;) 7!'&/2))$/ )$--.>?  !H FO FO  FO & & (    $ MM%rBprofilercKtjtddtjddt t j }tjdtjd |j|d {}|rtjd |y tjd y 75#t$r,}tjd t|d Yd }~y d }~wwxYww)z#Interactive profile creation wizardz[bold cyan]Create Browser Profile[/bold cyan] This will open a browser window for you to set up your identity. Log in to sites, adjust settings, then press 'q' to save.rrz[cyan]Enter profile name[/cyan]profile_)defaultz [cyan]Creating profile...[/cyan]zk[yellow]A browser window will open. After logging in to sites, press 'q' in this terminal to save.[/yellow]Nz0[green]Profile successfully created at:[/green] z$[red]Failed to create profile.[/red]z[red]Error creating profile: [/red]) rrUr r askrgtimecreate_profiler~r)r profile_name profile_pathrs r/create_profile_interactivers MM%R$*+, ::?8TWX\XaXaXcTdSeIfgL MM45 MMAF%44\BB  MML\N[ \ MM@ A C F 5c!fXVDEEFsNBDCC C6D7C D C D"C?:D?DDc Z|j}|stjdyt|t j dtt t|Dcgc]}t|dzc}d} t|dz }||}tj d|dd rP|j|d }|rtjd |dd ytjd |ddyycc}w#ttf$rtjdYywxYw)zInteractive profile deletionz-[yellow]No profiles found to delete.[/yellow]Nz,[red]Enter number of profile to delete[/red]r\FrrP show_choicesz.[red]Are you sure you want to delete profile 'rz'?[/red]r{z[green]Profile 'z' deleted successfully.[/green]z[red]Failed to delete profile 'z'.[/red]z[red]Invalid selection.[/red]) list_profilesrrUrr rrangelenrrgr delete_profilerp IndexError)rrridxrsuccesss r/delete_profile_interactivers%%'H  EF8$ **6#(X#78aQqS8  C 7#hl3- ;;GPVGXX`a b--gfo>G 00AA`ab ??PPXYZ c9  #7 567sC> >A!D D$D*)D*cVKtjd|d|dtdd|}t}t j dgdd } t |||dd {}|d k(r9tjtj|jd n|dk(rHtjtjtj|jd nb|dvr*tj|jjn4|dk(r/tj|jjddtjd||S7 #t $r+}tjdt#|Yd }~y d }~wwxYww)z(Use a profile to crawl a website via CLIz[cyan]Crawling [bold]z[/bold] using profile at [bold]z[/bold][/cyan]FT)headlessuse_managed_browser user_data_dirz[cyan]Output format[/cyan])allrlrDmdrrDrPrNrindentrlrDrrzNo title foundz$[green]Successfully crawled[/green] zed]Error crawling:[/red] )rrUrrr rrrldumps model_dumprmextracted_contentrD raw_markdownmetadatar:r~r)rrCrr output_formatrtrs r/crawl_with_profile_clirs] MM)#.Ml^[ijk  "K#$KJJ$:M "3 [$GG E ! MM$**V%6%6%8C D f $ MM$**TZZ0H0H%IRST U 0 0 MM&//66 7 g % MM&//--g7GH I  > > MM6 79 ?  #6 456sTA)D+C" D s[ H !  /  G  eL[abc!--/ @JC"czG3#:xTW[^T^5nquxnxdjEE MMAeWAcU"UG3ug> ? @ND4HRUV S=--/H "8 , s],X6 6 6 s] &x 0 s]&( ( ( s] MMA B   dA  7 )s$C.E0E1(EE8EEhelp_option_namesz-hz--help)context_settingscy)zICrawl4AI CLI - Web content extraction and browser profile management toolNrrBr/clirns rBbrowsercy)a(Manage browser instances for Crawl4AI Commands to manage browser instances for Crawl4AI, including: - status - Check status of the builtin browser - start - Start a new builtin browser - stop - Stop the running builtin browser - restart - Restart the builtin browser NrrrBr/ browser_cmdr ts rBstatusct} tj|j}|drm|d}tj t d|dd|dd|dd |d d tjd tj|d d ddytj t dddy#t$rA}tj dt|dtjdYd}~yd}~wwxYw)z"Show status of the builtin browserrunninginfoz:[green]Builtin browser is running[/green] CDP URL: [cyan]cdp_urlz[/cyan] Process ID: [yellow]pidz[/yellow] Browser type: [blue] browser_typez&[/blue] User data directory: [magenta]rz[/magenta] Started: [cyan]z%Y-%m-%d %H:%M:%S start_time[/cyan]zBuiltin Browser Statusrrzd[yellow]Builtin browser is not running[/yellow] Use 'crwl browser start' to start a builtin browserrz$[red]Error checking browser status: rr\N)ranyiorunget_builtin_browser_statusrrUr rr localtimer~rsysexit)rr rrs r/browser_status_cmdrs H8>>? ) &>D MM%""&y/!23''+E{m4''+N';&<=115o1F0GH""&--0CT^^TXYeTfEg"h!iip r /$   MM%F.%    2''+f-(0ud;7 D*   ))  + +       MM%"")+^^.$   MM%<-"  HHQK  =c!fXVLM  sA8D 6D E7EEstopctt} tj|j}|ds"tj t dddytj t dddtj|j}|r"tj t d dd ytj t d dd tjd y#t$rA}tj dt|dtjd Yd}~yd}~wwxYw)z Stop the running builtin browserr z8[yellow]No builtin browser is currently running[/yellow]zBuiltin Browser StoprrNz([cyan]Stopping builtin browser...[/cyan]rz3[green]Builtin browser stopped successfully[/green]rz)[red]Failed to stop builtin browser[/red]rr\z%[red]Error stopping builtin browser: r) rrrrrrUr kill_builtin_browserrrr~r)rr rrs r/browser_stop_cmdr+s H#8>>?i MM%J,%   e 6(  ))H99:  MM%E,$  MM%;,"  HHQK  =c!fXVLM  s%AC-A#C-66C-- D767D22D7viewz--urlz-uz,URL to navigate to (defaults to about:blank))rc Rt} tj|j}|ds"tj t dddy|d}|d}tj t d |d |xsd d dd ddl}ddl}tjdk(rdg}ntjdk(rdg}ndg}d|ddddg}|r|j| |j||ztj dy#t$rE} tj dt| dtj d|dYd} ~ yd} ~ wwxYw#t$rA} tj dt| dtjdYd} ~ yd} ~ wwxYw) z Open a visible window of the builtin browser This command connects to the running builtin browser and opens a visible window, allowing you to see what the browser is currently viewing or navigate to a URL. r zs[yellow]No builtin browser is currently running[/yellow] Use 'crwl browser start' to start a builtin browser firstzBuiltin Browser ViewrrNrrzR[cyan]Opening visible window connected to builtin browser[/cyan] CDP URL: [green]z[/green] URL to load: [yellow]z about:blank [/yellow]rrdarwinz>?i MM%L,%   f~y/ e&i($$'$8=#9 D)     <<8 #YZK \\W $WXK*+K't,<'=&> ? 2  (       $ K   [<7 8 MMa b K MM:3q6(&I J MM?yIIJ K K K  >? ) &>D $^ 4-. N MM%B/#  ii = =>G e?3!&  $U~'9'9.*'U 7~))&$7'389K9KJX\9]  e$$0>2''+f-(0ud;7 D,   ))  + +       MM%"")'3/$   MM%>/"  HHQK  ?AxvNO  sE)F-66F-- G767G22G7cdpz--user-data-dirz-dzGDirectory to use for browser data (will be created if it doesn't exist)z-Pz --headless)is_flagrz --incognitoz5Run in incognito/private mode (ignores user-data-dir)r incognitoct} d}|sr|rptjj|}tjj |s0t j d|dtj|d|}t j td|d|d|xsd d |rd nd d |rd nd d ddtj|j||||}|s+t j dtjdyy#t$rA} t j dt| dtjdYd} ~ yd} ~ wwxYw)aLaunch a standalone browser with CDP debugging enabled This command launches a browser with Chrome DevTools Protocol (CDP) debugging enabled, prints the CDP URL, and keeps the browser running until you press 'q'. The CDP URL can be used for various automation and debugging tasks. Examples: # Launch Chromium with CDP on default port 9222 crwl cdp # Use a specific directory for browser data and custom port crwl cdp --user-data-dir ~/browser-data --port 9223 # Launch in headless mode crwl cdp --headless # Launch in incognito mode (ignores user-data-dir) crwl cdp --incognito Nz[yellow]Directory 'z&' doesn't exist. Creating it.[/yellow]T)r%zH[cyan]Launching browser with CDP debugging[/cyan] Browser type: [green]r"z%[/yellow] User data directory: [cyan]zTemporary directoryz[/cyan] Headless: [cyan]r$r%z[/cyan] Incognito: [cyan]z5[/cyan] [yellow]Press 'q' to quit when done[/yellow]z CDP Browserrrz2[red]Failed to launch browser or get CDP URL[/red]r\z"[red]Error launching CDP browser: r)rrr{ expanduserr'rrUmakedirsr rrlaunch_standalone_browserrrr~r) rr rrr>rdata_dir expanded_pathrrs r/cdp_cmdrEsV8 H*]GG..}=M77>>-0 3M?Bhij MD9$H  e$$0>2''+f-**2*K6K)LM(0ud;< )2=>;  <   ))  . .       MMN O HHQK  :3q6(&IJ  sDD E7EEcrawl)requiredz--browser-configz-B)r'zBrowser config file (YAML/JSON))rrz--crawler-configz-CzCrawler config file (YAML/JSON)z--filter-configz-fzContent filter config filez--extraction-configz-ezExtraction strategy config filez--json-extractz-jFrRz;Extract structured data using LLM with optional description)r= flag_valuerrz--schemaz-szJSON schema for extractionz --browserz-Browser parameters as key1=value1,key2=value2)rcallbackrz --crawlerz-cz-Crawler parameters as key1=value1,key2=value2z--outputz-o)rrlrDr markdown-fitmd-fitr)rrz --output-filez-Oz"Output file path (default: stdout)z--bypass-cachez-bczBypass cache when crawling)r=rrz --questionz-qz(Ask a question about the crawled contentz --verbosez-v)r=z --profilez(Use a specific browser profile (by name)z --deep-crawl)bfsdfs best-firstzFEnable deep crawling with specified strategy (bfs, dfs, or best-first)z --max-pages z3Maximum number of pages to crawl in deep crawl modebrowser_configcrawler_config filter_configextraction_config json_extractschemaroutput output_file bypass_cachequestionr deep_crawl max_pagesc $|rt}|j|}|sM|j}|r%tj d|dt |ytj dy|si}||d<d|d<| rtj d| t jt|}tjt|}|r|jd;i|}|r|jd;i|}|s| d vr|r t|}n |s | d vrd d d d }ddk(rMtt|jd|jdd|jdd|_nC|dd k(r;tt|jd|jdd |_|Ot!\}}d}d|z}|r|n|}t#t%|||t'|ddd| |_| dk(rd} n |r t|}t'|}|jdst+j,d|dd vrt+j,d!|d|dd"k(rp|jd#r|jd$st+j,d%t#d;t%|d#|d$|d&|d'|jd(i|_n3|dd)k(rt/|*|_n|dd+k(rt1|*|_| rt2j4|_t9|_|re|d,k(rt=d-|.|_n/|d/k(rtAd-|.|_n|d0k(rtCd-|.|_| rtj d1|d2|d3tE}|jd4d|_#|jd4d|_#tIjJtL|||| }tO|tPr,tS|d5k(rt+jTd6y|d5} |}!n|} |g}!| rBt!\}}| jVjX}"tIjJtZ||"| ||y| sW| dk(rtO|tPrH|!D#cgc]}#|#j]}$}#t+jTt_j`|$d78yt+jTt_j`| j]d78y| dk(r_t | jbt_jd| jb}%t+jTt_j`|%d78y| d9vr*t+jT| jVjXy| d vrt+jT| jVjfy| dk(rti| d:5}&tO|tPrD|!D#cgc]}#|#j]}$}#|&jkt_j`|$d78n4|&jkt_j`| j]d78dddy| dk(r1ti| d:5}&|&jk| jbdddy| d9vr;ti| d:5}&|&jk| jVjXdddy| d vr;ti| d:5}&|&jk| jVjfdddyyycc}#wcc}#w#1swYyxYw#1swYyxYw#1swYyxYw#1swYyxYw#tl$r#}'t+j,to|'d}'~'wwxYw)L*62K ##F+**+JKKF#+LL**-F|TZG[F\+]^^F#u,#'' 3<;K;KK;X../kll2G3(,z2JVbcnVop ,] ;&3#&&x4 3 / f%32K&3 /f%52M&3 / %.%5%5K "(?(A % U"2F'3 /u$2F'3 /|+2K'3 /  FzlRabkallrst"$$jjE: $jjE:  %yy        fd #6{a BC )K K!K!(K .0OHe"++88H II)3(He T fd+;F"Ga1<<>"GK"GJJtzz+a@AJJtzz+*@*@*B1MN6!k334"&**[-J-J"K 4::oa@A-- ;//<<=55 ;//<<=+s+Pq!&$/?J&K!q||~&K &K ;q AB ;+A+A+CA NO PP 6!+s+;qGGK99:;;--+s+?qGGK00==>??55+s+?qGGK00==>??6!6#H"'LPP;;???? +""3q6**+sN]#%A]#6]#\)),]#8]#A#]#3-]#!.]#]#!\36\. A\3*]#3]#\? ]#)]#9&] ]#(]#8&]]#)]#.\33\<8]#<]#?]]#]# ]]#]#] ]# ]## ^,^  ^rcty)Show usage examplesN)rrrBr/ examples_cmdrs OrBcy)aManage global configuration settings Commands to view and update global configuration settings: - list: Display all current configuration settings - get: Get the value of a specific setting - set: Set the value of a specific setting NrrrBr/ config_cmdrs rBrcbt}tdddd}|jdd|jd d |jd d |jd dtjD]\}}|j ||d}|}|j ddr|rd}|ddk(r6t |j}t |dj}nt |d}|j|t |||dtj|y)zList all configuration settingszCrawl4AI ConfigurationTrrrSettingrrValuerDefaultr DescriptionwhitersecretF********rboolean descriptionN) r0rrrrr:rrerrrU)r1rrsettingrs display_value default_values r/config_list_cmdrs/ F 0dQ\kq rE Yf- WG, Yh/ ]'2%++- W 3 23 ;;x 'E&M 6?i 'J,,.M 2399;M 23M     M "   , MM%rBr:rcpt}|j}|tvrtj d|dy|j |t|d}|}t|j ddr|rd}tj d|d |d tj d t|d d y)z$Get a specific configuration setting[red]Error: Unknown setting ''[/red]NrrFrz[cyan][/cyan] = [green][/green]z[dim]Description: rz[/dim])r0upperrrrUr:)rr1rsrs r/config_get_cmdrs F ))+C - 5cU'BC JJsM#.y9 :EMSh.5"  MMF3%0xHI MM&}S'9-'H&IPQrBsetrsct}|j}|tvrTtj d|dtj ddj tj dyt|}|ddk(r@|jd vrd }nl|jd vrd }nWtj d y|ddk(r9|}d|vr3||dvr,tj ddj |ddy||<t||}|jdd r|rd}tj d|d|dy)zSet a configuration settingrrz[yellow]Available settings: z, r.Nrr)r]yesryT)r^no0nFz?[red]Error: Invalid boolean value. Use 'true' or 'false'.[/red]stringrz"[red]Error: Value must be one of: rrrz&[green]Successfully set[/green] [cyan]rr) r0rrrrUrrrer5r:)rrsr1r typed_valuers r/config_set_cmdr3sT  F ))+C - 5cU'BC 4TYY}?Q?Q?S5T4UU^_`C Gv)# ;;=5 5K [[]7 7K MM[ ]  H $   E1C$C MM>tyyQZI[?\>]]cd e F3Kv M{{8U# "  MM:3%?PQ^P__ghirBc6tjty)zManage browser profiles interactively Launch an interactive browser profile manager where you can: - List all existing profiles - Create new profiles for authenticated browsing - Delete unused profiles N)rrrrrBr/ profiles_cmdr`s IIorB)rz --examplerz,Enable deep crawling with specified strategyexamplec|r ty|s8tj}tj|j ytj}|j t ||||||||| | | | | |||y)aCrawl4AI CLI - Web content extraction tool Simple Usage: crwl https://example.com Run with --example to see detailed usage examples. Other commands: crwl profiles - Manage browser profiles for identity-based crawling crwl crawl - Crawl a website with advanced options crwl cdp - Launch browser with CDP debugging enabled crwl browser - Manage builtin browser (start, stop, status, restart) crwl config - Manage global configuration settings crwl examples - Show more usage examples Configuration Examples: crwl config list - List all configuration settings crwl config get DEFAULT_LLM_PROVIDER - Show current LLM provider crwl config set VERBOSE true - Enable verbose mode globally crwl config set BROWSER_HEADLESS false - Default to visible browser N)rCrPrQrRrSrTrUrrrVrXrYrrrZr[)rr;get_current_contextr<get_helpinvoker)rCrrPrQrRrSrTrUrrrVrXrYrrrZr[rqs r/rrlsV '') 3<<>"  # # %CJJ %%#+!!#rBcddl}t|jdks|jdtjvr|jj ddty)Nrrr\rF)rrargvrcommandsinsert)rs r/mainrs? 388}qCHHQKs||; 7#ErB__main__)[r;rrrrtypingrrrrrlr*r rich.consoler rich.tabler rich.panelr rich.promptr r crawl4air r rrrrrrrrrrrrrrrcrawl4ai.configrlitellmrpathlibrrdictr0r5tuplerrArXrzrrboolrrrgrrrrrrrgrouprr commandroptionChoicer'r+r7r;rEargumentrrrrrrrrr__name__rrBr/rsb  ,, '&* ) '4 't %S/2 3 # c S Y\ 04S>2 P8C= PT P"8C="T" E3 E] EIY Edh EhTSS!T$sCx.%9!FFF. 7 7D'R 6D,` 2T84DEF G 9     X> W <5<<Y8O+PZd57 h3;[\ ($=[\7C7s7d7]]7 7r V''R V gt"PQB(3-BRBH Y <5<<Y8O+PZ^?A h3;ij ($=[\LhsmL8C=LT\]aTbL]kA L\U ,uv h3;[\ <5<<Y8O+PZd57 lD/MN mT0ghA8C=AA3ARVAcgAiO7]wAHW%  $ZUZZt-DKlm  $ZUZZt-DKlm JEJJd,CJfg #T  $0GNop eDXUV j$ZUZZt%Ai+LPi+[_i+i+%(i+8<i+HKi+VZi+ehi+vyi+GJi+o^R/Sfc||aVqhnn&$i+VZ8   F  D E%RR&R* E%$'(j(jC(j(&(jTZ  "& k4.CD  $ZUZZt-DKlm  $ZUZZt-DKlm JEJJd,CJfg #T  $0GNop eDXUV j$ZUZZt%7IM7X\77#'7367AE7PS7ad7qt7oDR/S_||aVqhnnE'$7r  zFrB