
    Mi	                         S SK JrJr  S SKJrJrJr  S SKrS SKrS SK	J
r
  S SKr\R                  " \5      r " S S\5      r " S S5      rg)	    )ABCabstractmethod)DictTypeUnionN)Pathc                   N   ^  \ rS rSrS r\SS\S\4S jj5       rU 4S jrSr	U =r
$ )	BaseCrawler   c                 b    [         R                  " U R                  R                  5      U l        g N)logging	getLogger	__class____name__logger)selfs    f/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/hub.py__init__BaseCrawler.__init__   s    ''(?(?@    urlreturnc                    #    g7f)zb
Implement this method to return JSON string.
Must accept URL + arbitrary kwargs for flexibility.
N )r   r   kwargss      r   runBaseCrawler.run   s
      	s   c                   > [         TU ]  " S0 UD6  U R                  nUR                  R                  S:  d  [        U R                   S35      e[        R                  " U5      (       d  [        U R                   S35      eg)z+Enforce interface validation on subclassing   z/ must implement 'run(self, url: str, **kwargs)'z.run must be asyncNr   )	super__init_subclass__r   __code__co_argcount	TypeErrorr   inspectiscoroutinefunction)clsr   
run_methodr   s      r   r"   BaseCrawler.__init_subclass__   sx    !+F+ WW
""..!3s||n,[\]] **:66s||n,>?@@ 7r   )r   ) )r   
__module____qualname____firstlineno__r   r   strr   r"   __static_attributes____classcell__)r   s   @r   r
   r
      s7    A S C  A Ar   r
   c                       \ rS rSr% 0 r\\\\   4   \	S'   \
S 5       r\
S\4S j5       r\
S\S\\\   S4   4S j5       rS	rg)

CrawlerHub%   	_crawlersc           
         [        [        5      R                  S-  nUR                  5        Hv  nUR	                  5       (       d  M   [
        R                  " SUR                   S35      n[        U5       H(  nU R                  [        X45      UR                  5        M*     Mx     g! [         a9  n[        R                  SUR                   S[        U5       35         SnAM  SnAff = f)z3Dynamically load crawlers from /crawlers in 3 linescrawlerszcrawl4ai.crawlers.z.crawlerzFailed z: N)r   __file__parentiterdiris_dir	importlibimport_modulenamedir_maybe_register_crawlergetattr	Exceptionr   warningr/   )r(   	base_pathcrawler_dirmoduleattres         r   _discover_crawlersCrawlerHub._discover_crawlers(   s     N))J6	$,,.K!!##	K&44,[-=-=,>hGF !$F33#F1;3C3C !, / ! KNNW[-=-=,>bQ#IJJKs   AB((
C+2.C&&C+r>   c                     [        U[        5      (       ab  [        U[        5      (       aL  U[        :w  aA  [        R
                  " UR                  5      n[        US0 5      Ul        XR                  U'   gggg)z Brilliant one-liner registration__meta__N)

isinstancetype
issubclassr
   r<   r=   r,   rA   metar5   )r(   objr>   rF   s       r   r@   "CrawlerHub._maybe_register_crawler9   sa     c4  Z[%A%Ac[FX,,S^^<Fvz26CH"%MM$ GY%A r   r   Nc                 z    U R                   (       d  U R                  5         U R                   R                  U5      $ r   )r5   rI   get)r(   r>   s     r   rT   CrawlerHub.getA   s*    }}""$}}  &&r   r   )r   r,   r-   r.   r5   r   r/   r   r
   __annotations__classmethodrI   r@   r   rT   r0   r   r   r   r3   r3   %   s{    .0ItCk**+0K K  & & & 's 'uT+%6%<= ' 'r   r3   )abcr   r   typingr   r   r   r   r<   pathlibr   r&   r   r   r   r
   r3   r   r   r   <module>r[      sB    # $ $    			8	$A# A2 '  'r   