/i$FddlmZddlmZddlZddlZddlZddlddlZddl m Z ejjejjejejjeZedZedZed Zd Zed Zed Zedd efdZedZedZedZedZddZdZe dk(reyy)) lru_cache)PathN)*)MODEL_REPO_BRANCHcddl}|jdk(r%|jj|jS|jdk(ryy)Nrcudampsl0)torchtyper get_device_properties total_memory)devicer s Q/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/crawl4ai/model_loader.pyget_available_memoryr s> {{fzz//7DDD  c|t|}|jdk(ry|jdvr|dk\ry|dk\ry|dk\ry y y) Ncpu)r r lll@ )rr )ravailable_memorys rcalculate_batch_sizersO+F3 {{e  ' | +  -  ,rcddl}|jjr|jd}|S|jj jr|jd}|S|jd}|S)Nrr r r)r r is_availablerbackendsr )r rs r get_devicer,sl zz f% M    ( ( *e$ Me$ Mrc@t}|j|||fS)N)rto)modelrs rset_model_devicer#9s \F HHV &=rc(tjjtjdt j d}tj |dtj |ddtj |dd|S)NCRAWL4_AI_BASE_DIRECTORYz .crawl4aiT)exist_okz/cachez/models)ospathjoingetenvrhomemakedirs) home_folders rget_home_folderr.?sk'',, ,diik:KKKK d+KK;-v&6KK;-w'$7 rcddlm}m}|jdd}|jdd}|j t |\}}||fS)Nr) BertTokenizer BertModelzbert-base-uncasedresume_download) transformersr0r1from_pretrainedevalr#)r0r1 tokenizerr"rs rload_bert_base_uncasedr8JsT5--.ASW-XI  % %&94 % PE JJL$U+ME6 e rreturncddlm}m}|j|d}|j|d}|j t |\}}||fS)zLoad the Hugging Face model for embedding. Args: model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5". Returns: tuple: The tokenizer and model. r) AutoTokenizer AutoModelNr2)r4r;r<r5r6r#) model_namer;r<r7r"rs rload_HF_embedding_modelr>UsS6--j$-OI  % %j$ % GE JJL$U+ME6 e rcddlm}m}ddlm}|j d}|j d}|j t |\}}|d||}|S)Nr)r;"AutoModelForSequenceClassification)pipelinez1dstefa/roberta-base_topic_classification_nyt_newsztext-classification)r"r7)r4r;r@rAr5r6r#)r;r@rAr7r"rpipes rload_text_classifierrChs]N%--;I / > >; E JJL$U+ME6 )) LD Krc ddlm}m}ddlmddl d}|j |d|j |djt\jjd fd }|fS)Nr)r@r;)expitzcardiffnlp/tweet-topic-21-multir2c|ddd|}|jDcic]\}}||j}}}j5di|}dddjj j j }|}||k\dz}g} |D]9} t| D cgc]\} } | dk(s | } } } | j| ;| Scc}}w#1swYxYwcc} } w)NptT)return_tensorspadding truncation max_lengthr) itemsr!no_gradlogitsdetachrnumpy enumerateappend)texts thresholdrKtokenskeyvaloutputscores predictions batch_labels predictionivaluelabels class_mappingrrEr"r7r s r _classifierz4load_text_multilabel_classifier.._classifiers !  17  $,CC   ]]_ %_V_F %%%'++-335v*a/  % (J1::1F%-Q%ST* a F    '  ( %  % %sC* C0= C< C<0C9)g?r) r4r@r; scipy.specialrEr r5r6r#configid2label) r@r;MODELrbrarrEr"r7r s @@@@@@rload_text_multilabel_classifierrgysN# .E--eT-JI . > > t ? E JJL$U+ME6LL))M8  rcddl} |jjd|jjdS#t$r|j dY7wxYw)Nrztokenizers/punktpunkt)nltkdatafind LookupErrordownload)rjs rload_nltk_punktrosN )* 99>>, --  gs<AAc 6ddl}d}t}t||z }|jrt |j sd}t }t|dz }td|jr; tj||jrtj| tjdd d ||t|gtjtjd t|d z }|jd d |d z dz }tj ||tj|td |j't|S#t$r*tdtd|td|YywxYw#tj"$r} td| Yd} ~ yd} ~ wt$$r} td| Yd} ~ yd} ~ wwxYw#t$$r} td| Yd} ~ yd} ~ wwxYw)Nrmodels/reutersz)https://github.com/unclecode/crawl4ai.gitcrawl4aiu7[LOG] ⏬ Downloading Spacy model for the first time...zh[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:z- gitclonez-bT)stdoutstderrcheckmodels)parentsr&reutersu-[LOG] ✅ Spacy Model downloaded successfullyz0An error occurred while cloning the repository: zAn error occurred: zError loading spacy model: )spacyr.rexistsanyiterdirrprintshutilrmtreePermissionError subprocessrunstrDEVNULLmkdircopytreeCalledProcessError Exceptionload) r{namer- model_folderrepo_urlbranch repo_folder models_folder source_folderes rload_spacy_modelrs D!#K $t+L    !c,*>*>*@&A>";'*4  GH      k*&&(MM,/  NNvx[9IJ!))!))  !-8M   t  <((2Y>M OOM< 8 MM+ & A Bzz#l+,,K# ~;-()<.)*  :,,  DQCH I  's+ ,   +A3/0sO;:E46B#F*G740F'&F'*G4=G G4G//G47 HHHc|rtdt}tjj |dtjj |dg}|D]1}t |j stj|3tdtdt\}}td|tdttdy ) z*Download all models required for Crawl4AI.z![LOG] Removing existing models...rqrxz[LOG] Existing models removed.z$[LOG] Downloading text classifier...z [LOG] Text classifier loaded on z,[LOG] Downloading custom NLTK Punkt model...u-[LOG] ✅ All models downloaded successfully.N) rr.r'r(r)rr|rrrgro)remove_existingr- model_foldersfolder_rs rdownload_all_modelsrs 12%' GGLL&6 7 GGLLh / $ &FF|""$ f% & ./ 01/1IAv ,VH 56 89 9:rctdtdtjd}|jddd|j }t |j y) Nz/[LOG] Welcome to the Crawl4AI Model Downloader!zE[LOG] This script will download all the models required for Crawl4AI.zCrawl4AI Model Downloader) descriptionz--remove-existing store_truez)Remove existing models before downloading)actionhelp)r)rargparseArgumentParser add_argument parse_argsrr)parserargss rmainrs_ ;< QR  $ $1L MF  8    D(<(<=r__main__)zBAAI/bge-small-en-v1.5)F)! functoolsrpathlibrrr'r model_loaderrcrawl4ai.configrr(realpathr)getcwddirname__file__ __location__rrrr#r.r8tupler>rCrgrorrr__name__rLrrrsZ -ww YRYY["''//(:S TU     &          E $      3 3l . . = =@;: > zFr