
    /i	                         d dl mZmZ d dlmZmZmZ d dlZd dlZd dl	m
Z
 d dlZ ej                  e      Z G d de      Z G d d      Zy)	    )ABCabstractmethod)DictTypeUnionN)Pathc                   @     e Zd Zd Zeddedefd       Z fdZ xZS )BaseCrawlerc                 `    t        j                  | j                  j                        | _        y N)logging	getLogger	__class____name__logger)selfs    H/mnt/e/genesis-system/.venv/lib/python3.12/site-packages/crawl4ai/hub.py__init__zBaseCrawler.__init__   s    ''(?(?@    urlreturnc                    K   yw)zz
        Implement this method to return JSON string.
        Must accept URL + arbitrary kwargs for flexibility.
        N )r   r   kwargss      r   runzBaseCrawler.run   s      	s   c                     t        |   di | | j                  }|j                  j                  dk\  st        | j                   d      t        j                  |      st        | j                   d      y)z+Enforce interface validation on subclassing   z/ must implement 'run(self, url: str, **kwargs)'z.run must be asyncNr   )	super__init_subclass__r   __code__co_argcount	TypeErrorr   inspectiscoroutinefunction)clsr   
run_methodr   s      r   r   zBaseCrawler.__init_subclass__   su    !+F+ WW
""..!3s||n,[\]] **:6s||n,>?@@ 7r   ) )	r   
__module____qualname__r   r   strr   r   __classcell__)r   s   @r   r
   r
      s7    A S C  A Ar   r
   c                   |    e Zd ZU i Zeeee   f   ed<   e	d        Z
e	defd       Ze	dedeee   df   fd       Zy)
CrawlerHub	_crawlersc           
         t        t              j                  dz  }|j                         D ]m  }|j	                         s	 t        j                  d|j                   d      }t        |      D ](  }| j                  t        ||      |j                         * o y# t        $ r8}t        j                  d|j                   dt        |              Y d}~d}~ww xY w)z3Dynamically load crawlers from /crawlers in 3 linescrawlerszcrawl4ai.crawlers.z.crawlerzFailed z: N)r   __file__parentiterdiris_dir	importlibimport_modulenamedir_maybe_register_crawlergetattr	Exceptionr   warningr*   )r%   	base_pathcrawler_dirmoduleattres         r   _discover_crawlerszCrawlerHub._discover_crawlers(   s     N))J6	$,,. 	KK!!#	K&44,[-=-=,>hGF !$F 33#FD1;3C3C	K ! KNNW[-=-=,>bQ#IJJKs   AB	C'.CCr7   c                     t        |t              r\t        |t              rK|t        k7  rAt	        j
                  |j                        }t        |di       |_        || j                  |<   yyyy)z Brilliant one-liner registration__meta__N)

isinstancetype
issubclassr
   r5   r6   r(   r:   metar.   )r%   objr7   r?   s       r   r9   z"CrawlerHub._maybe_register_crawler9   s]     c4 Z[%Ac[FX,,S^^<Fvz26CH"%CMM$ GY%A r   r   Nc                 p    | j                   s| j                          | j                   j                  |      S r   )r.   rB   get)r%   r7   s     r   rK   zCrawlerHub.getA   s*    }}""$}}  &&r   )r   r(   r)   r.   r   r*   r   r
   __annotations__classmethodrB   r9   r   rK   r   r   r   r-   r-   %   s{    .0ItCk**+0K K  & & & 's 'uT+%6%<= ' 'r   r-   )abcr   r   typingr   r   r   r   r5   pathlibr   r#   r   r   r   r
   r-   r   r   r   <module>rQ      sD    # $ $    			8	$A# A2 '  'r   