
    Miv                        S SK JrJrJrJrJr  SSKJr  SSKJ	r	J
r
JrJr  SSKJr  SSKJr  S SKJr  S SKrS SKrS SKrS SKrS S	KJr  S SKrS S
KJrJr  SSKJr   " S S5      r " S S\5      r  " S S\ 5      r! " S S\ 5      r"g)    )DictOptionalListTupleUnion   )CrawlerRunConfig)CrawlResultCrawlerTaskResultCrawlStatusDomainState)CrawlerMonitor)AsyncWebCrawler)AsyncGeneratorN)urlparse)ABCabstractmethod)get_true_memory_usage_percentc            
           \ rS rSr    SS\\\4   S\S\S\\   4S jjrS\	S	\	4S
 jr
S\	S	S4S jrS\	S\S	\4S jrSrg)RateLimiter   N
base_delay	max_delaymax_retriesrate_limit_codesc                 Z    Xl         X l        X0l        U=(       d    SS/U l        0 U l        g )Ni  i  )r   r   r   r   domains)selfr   r   r   r   s        s/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/async_dispatcher.py__init__RateLimiter.__init__   s,     %"& 0 >S#J/1    urlreturnc                 ,    [        U5      R                  $ N)r   netloc)r   r#   s     r   
get_domainRateLimiter.get_domain*   s    }###r"   c                 8  #    U R                  U5      nU R                  R                  U5      nU(       d&  [        5       U R                  U'   U R                  U   n[        R                  " 5       nUR
                  (       aI  [        SUR                  XCR
                  -
  -
  5      nUS:  a  [        R                  " U5      I S h  vN   UR                  S:X  a"  [        R                  " U R                  6 Ul        [        R                  " 5       Ul        g  NQ7fNr   )r(   r   getr   timelast_request_timemaxcurrent_delayasynciosleeprandomuniformr   )r   r#   domainstatenow	wait_times         r   wait_if_neededRateLimiter.wait_if_needed-   s     %  (#.=DLL LL(Eiik""Au22c<S<S6STUI1}mmI... !#"(..$//"BE"&))+ /s   CDDADstatus_codec                    U R                  U5      nU R                  U   nX R                  ;   aq  U=R                  S-  sl        UR                  U R                  :  a  g[        UR                  S-  [        R                  " SS5      -  U R                  5      Ul        g[        [        R                  " U R                  6 UR                  S-  5      Ul        SUl        g)Nr   F   g      ?g      ?r   T)r(   r   r   
fail_countr   minr0   r3   r4   r   r/   r   )r   r#   r;   r5   r6   s        r   update_delayRateLimiter.update_delayA   s    %V$///!$"2"22 #&##a'&..t*DDdnn#E  #&0%2E2E2L#E  !Er"   )r   r   r   r   r   ))      ?g      @g      N@   N)__name__
__module____qualname____firstlineno__r   floatintr   r    strr(   r9   boolr@   __static_attributes__ r"   r   r   r      s     +5&*2%,'2 2 	2
 s)2$c $c $. . .( # $ r"   r   c                       \ rS rSr  SS\\   S\\   4S jjrS\S\	\
\\
   4   S\\
   4S	 jr\ SS\S
\	\
\\
   4   S\S\\   S\4
S jj5       r\ SS\\   S\S
\	\
\\
   4   S\\   S\\   4
S jj5       rSrg)BaseDispatcherY   Nrate_limitermonitorc                 F    S U l         0 U l        SU l        Xl        X l        g r+   )crawler_domain_last_hitconcurrent_sessionsrQ   rR   )r   rQ   rR   s      r   r    BaseDispatcher.__init__Z   s&    
 24#$ (r"   r#   configsr$   c                     [        U[        5      (       a  U$ U(       d  gU H  nUR                  U5      (       d  M  Us  $    g)zSelect the appropriate config for a given URL.

Args:
    url: The URL to match against
    configs: Single config or list of configs to choose from
    
Returns:
    The matching config, or None if no match found
N)
isinstancer	   is_match)r   r#   rX   configs       r   select_configBaseDispatcher.select_confige   sD     g/00N  Fs## 
 r"   r\   task_idc                    #    g 7fr&   rM   )r   r#   r\   r_   rR   s        r   	crawl_urlBaseDispatcher.crawl_url   
      	   urlsrT   c                    #    g 7fr&   rM   )r   re   rT   r\   rR   s        r   run_urlsBaseDispatcher.run_urls   rc   rd   )rU   rV   rT   rR   rQ   )NNr&   )rD   rE   rF   rG   r   r   r   r    rJ   r   r	   r   r]   r   r   ra   r   rg   rL   rM   r"   r   rO   rO   Y   s%    /3,0	{+	 .)	 u5EtL\G]5]/^ ckl|c} 4  -1 &-=(>>? 	
 .) 
   -13i ! &-=(>>?	
 .) 
	  r"   rO   c                   >  ^  \ rS rSr         SS\S\S\S\S\S\S	\\   S
\\   S\\   4U 4S jjjr	S r
S\S\S\4S jr SS\S\\\\   4   S\S\S\4
S jjrS\\   S\S\\\\   4   S\\   4S jrS rS\\   S\S\\\\   4   S\\S4   4S jrSrU =r$ )MemoryAdaptiveDispatcher   Nmemory_threshold_percentcritical_threshold_percentrecovery_threshold_percentcheck_intervalmax_session_permitfairness_timeoutmemory_wait_timeoutrQ   rR   c
                 
  > [         T
U ]  X5        Xl        X l        X0l        X@l        XPl        X`l        Xpl        [        R                  " 5       U l        [        R                  " 5       U l        SU l        SU l        S U l        g )NF        )superr    rl   rm   rn   ro   rp   rq   rr   r1   Queueresult_queuePriorityQueue
task_queuememory_pressure_modecurrent_memory_percent_high_memory_start_time)r   rl   rm   rn   ro   rp   rq   rr   rQ   rR   	__class__s             r   r    !MemoryAdaptiveDispatcher.__init__   sr     	/(@%*D'*D',"4 0#6 #MMO!//1$)!&)#8<$r"   c                 (  #     [        5       U l        U R                  U R                  :  a  U R                  (       dO  SU l        [        R                  " 5       U l        U R                  (       a  U R                  R                  S5        GOU R
                  c  [        R                  " 5       U l        U R                  bV  U R
                  bI  [        R                  " 5       U R
                  -
  U R                  :  a  [        SU R                   S35      eOU R                  (       aU  U R                  U R                  ::  a;  SU l        SU l        U R                  (       a  U R                  R                  S5        O!U R                  U R                  :  a  SU l        U R                  U R                  :  a,  U R                  (       a  U R                  R                  S5        [        R                  " U R                  5      I Sh  vN   GM   N7f)	zEBackground task to continuously monitor memory usage and update stateTPRESSURENz$Memory usage exceeded threshold for z secondsFNORMALCRITICAL)r   r{   rl   rz   r-   r|   rR   update_memory_statusrr   MemoryErrorrn   rm   r1   r2   ro   )r   s    r   _memory_monitor_task-MemoryAdaptiveDispatcher._memory_monitor_task   s    *G*ID' **d.K.KK0004D-3799;D0||99*E33;7;yy{400< 88D IIK$*F*FF$JbJbb)  $ 8 89C  **t/J/JdNmNm/m,1)/3,<<LL55h?,,t/L/LL/3, **d.M.MM<<LL55jA -- 3 3444M L 5s   HHH	Hr8   retry_countr$   c                 *    XR                   :  a  U* $ U$ )zCalculate priority score (lower is higher priority)
- URLs waiting longer than fairness_timeout get higher priority
- More retry attempts decreases priority
)rq   )r   r8   r   s      r   _get_priority_score,MemoryAdaptiveDispatcher._get_priority_score   s    
 ,,,:r"   r#   r\   r_   c                 
  #    [         R                   " 5       nSnS=pxU R                  X5      n	U	cp  SU 3nU R                  (       a)  U R                  R                  U[        R
                  US9  [        UU[        USSS0SUS9S	S	U[         R                   " 5       UUS
9	$ [        R                  " 5       n
U
R                  5       R                  S-  n U R                  (       a*  U R                  R                  U[        R                  UUS9  U =R                  S-  sl        U R                  (       a#  U R                  R                  U5      I S h  vN   U R                   U R"                  :  Ga  [         R                   " 5       nU R%                  X-
  US-   5      nU R&                  R)                  XX4S-   U445      I S h  vN   U R                  (       a)  U R                  R                  U[        R*                  SS9  [        UU[        USSS0SSS9S	S	U[         R                   " 5       SUS-   S
9	[         R                   " 5       nU R                  (       a  U R                  R                  UUUUUUS9  U =R                  S-  sl        $ U R,                  R/                  XUS9I S h  vN nU
R                  5       R                  S-  nUU-
  =pxU R                  (       a  UR0                  (       a{  U R                  R3                  XR0                  5      (       dQ  S[5        U5      R6                   3nU R                  (       a(  U R                  R                  U[        R
                  S9  UR8                  (       dF  UR:                  nU R                  (       a(  U R                  R                  U[        R
                  S9  O9U R                  (       a(  U R                  R                  U[        R<                  S9  [         R                   " 5       nU R                  (       a  U R                  R                  UUUUUUS9  U =R                  S-  sl        [        UUUUUUUUUS
9	$  GN GN GN! [>         ad  n[A        U5      nU R                  (       a(  U R                  R                  U[        R
                  S9  [        US0 S[A        U5      S9n S nANS nAff = f! [         R                   " 5       nU R                  (       a  U R                  R                  UUUUUUS9  U =R                  S-  sl        f = f7f)N rt   )No matching configuration found for URL: statuserror_messager   no_config_matchFr#   htmlmetadatasuccessr   r   )	r_   r#   resultmemory_usagepeak_memory
start_timeend_timer   r      )r   r   r   r   z(Requeued due to critical memory pressurerequeued)r   r   r   r   r   r\   
session_id+Rate limit retry count exceeded for domain r   )!r-   r]   rR   update_taskr   FAILEDr   r
   psutilProcessmemory_inforssIN_PROGRESSrV   rQ   r9   r{   rm   r   ry   putQUEUEDrT   arunr;   r@   r   r'   r   r   	COMPLETED	ExceptionrJ   )r   r#   r\   r_   r   r   r   r   r   selected_configprocessstart_memoryenqueue_timepriorityr   r   
end_memoryes                     r   ra   "MemoryAdaptiveDispatcher.crawl_url   s     YY[
%(( ,,S9 "GuMM||((&--"/ )  %"&(9:!"/ %+' & .."**,00K@V	*||((&22) +	 )  $$)$  ''66s;;; **d.M.MM#yy{33L4M{]^_oo))87RSOUa5b*cddd <<LL,,*11&P -  )#&bHj3I %5_ "# !)!YY["L +a\ yy{H||((%!- +"/ + )  $$)$Q  <<,,SU\,]]F !,,.22kBJ)3l)BBL   V%7%7((55c;M;MNN&QRZ[^R_RfRfQg$hM||00ASAS0T >> & 4 4<<LL,,W[=O=O,P((9N9N(O yy{H||((%!- +"/ + )  $$)$ %#!'#

 
	
Y < e6 ^*  	FM||((9K9K(L b2uCPQFF		 yy{H||((%!- +"/ + )  $$)$s   CUA?Q5 Q,A0Q5 Q/A1Q5 6AUQ5 -Q2.DQ5 S& 	9Q5 A*U,Q5 /Q5 2Q5 5
S#?ASS& S##S& &AUUre   rT   c           	        #    X l         [        R                  " U R                  5       5      nU R                  (       a  U R                  R                  5         / n U H  n[        [        R                  " 5       5      nU R                  (       a  U R                  R                  Xv5        U R                  R                  SXgS[        R                  " 5       445      I S h  vN   M     / nU R                  R                  5       (       a  U(       Ga  UR                  5       (       a2  UR                  5       n	U	(       a  U H  n
U
R!                  5         M     U	eU R"                  (       d  U R$                  ['        U5      -
  nUS:  a   U R                  R)                  5       u  nu  pgp[        R                  " U R+                  XcX}5      5      nUR-                  U5        U R                  (       aA  [        R                  " 5       U-
  nU R                  R/                  UU[0        R2                  S9  US-  nUS:  a  M  U(       a_  [        R6                  " US[        R8                  S9I S h  vN u  nnU H  nUI S h  vN nUR-                  U5        M      [;        U5      nO+[        R<                  " U R>                  S-  5      I S h  vN   U RA                  5       I S h  vN   U R                  R                  5       (       d  GM  U(       a  GM  UR!                  5         U R                  (       a  U R                  RG                  5         U$  GNq! [        R4                   a     GM-  f = f GN N N N! [B         a>  nU R                  (       a'  U R                  RE                  S[        U5       35        e S nAff = f! UR!                  5         U R                  (       a  U R                  RG                  5         f f = f7f)Nr   r8   r   r   皙?timeoutreturn_whenr=   QUEUE_ERROR: )$rT   r1   create_taskr   rR   startrJ   uuiduuid4add_taskry   r   r-   emptydone	exceptioncancelrz   rp   len
get_nowaitra   appendr   r   r   
QueueEmptywaitFIRST_COMPLETEDlistr2   ro   _update_queue_prioritiesr   r   stop)r   re   rT   r\   memory_monitorresultsr#   r_   active_tasksexctslotsr   r   r   taskr8   r   pendingcompleted_taskr   r   s                         r   rg   !MemoryAdaptiveDispatcher.run_urlsv  s-      !,,T-F-F-HI<<LL N	$djjl+<<LL))'7oo))1sQ		.L*MNNN  L oo++--!&&(((224C!-AHHJ ".!	 00 33c,6GGE!)"RVRaRaRlRlRnOH&Os[ $+#6#6 $sG Q$D )//5  $||,0IIK,,F	 $ 8 8$+.7+6+B+B !9 !" "QJE)  !)6  *1,,$cw?V?V+ %MD'
 +/'5!5v. +/
 $(=L "--(;(;a(?@@@ 33555o oo++--@ !!#||!!#Q OL  '11 "!"% "6 A 6 	||11M#a&2JK	 !!#||!!# s   APBM9  M!B'M9 	B.M 7M9 ?.M9 -M0.M9 ?M3 AM9 M5M9 #M7$#M9 
M9 =PM9 M-(M9 ,M--M9 3M9 5M9 7M9 9
O9N<<OO >PPc                 v  #    U R                   R                  5       (       a  g/ n [        R                  " 5       nU R                   R                  5       (       Gd  [        R                  " 5       U-
  S:  a   [        R                  " U R                   R                  5       SS9I Sh  vN u  nu  pEpg[        R                  " 5       nX-
  n	U R                  X5      n
UR                  XXVU445        U R                  (       a2  XPR                  R                  ;   a  U R                  R                  XYS9  U R                   R                  5       (       d  [        R                  " 5       U-
  S:  a  M  U(       a  U R                  (       a  [        U5      nU Vs/ s H
  oS   S   PM     nnU(       a!  [        R                  " 5       [!        U5      -
  OS	nU(       a  [#        S
 U 5       5      [        U5      -  OS	nU R                  R%                  UUUS9  UR'                  S S9  U H&  nU R                   R)                  U5      I Sh  vN   M(     g GN! [        R                   a     M  f = f! [         a2  nU R                  R                  S[        U5       35         SnAGN8SnAff = fs  snf  Nk7f)zJPeriodically update priorities of items in the queue to prevent starvationNg      @r   )r   )r8   r   r   rC   r   c              3   R   #    U  H  n[         R                   " 5       U-
  v   M     g 7fr&   )r-   ).0r   s     r   	<genexpr>DMemoryAdaptiveDispatcher._update_queue_priorities.<locals>.<genexpr>  s     DA		as   %')total_queuedhighest_wait_timeavg_wait_timec                     U S   $ r+   rM   )xs    r   <lambda>CMemoryAdaptiveDispatcher._update_queue_priorities.<locals>.<lambda>  s    adr"   )key)ry   r   r-   r1   wait_forr,   r   r   rR   statsr   TimeoutErrorr   r   rJ   r   r?   sumupdate_queue_statisticssortr   )r   
temp_itemsdrain_startr   r#   r_   r   r   current_timer8   new_priorityr   r   item
wait_timesr   r   s                    r   r   1MemoryAdaptiveDispatcher._update_queue_priorities  s?     ??  "" 
	H))+Koo++--$))+2Kc2QPWP`P`++-sQ KGHGs[
 $(99;L , ;I#'#;#;I#SL %%|7Q]5^&_` ||<<3E3E(E000N# oo++--$))+2Kc2Q6 $,,z?L1;<q'!*J<AK		c*o =QRZdCDDDs:VjkM LL00)"3+ 1  	N+ D//%%d+++ SK  ++   	HLL--c!fX.FGG	H =  ,s   #J9AI3 80I (I)BI 5;I3 2'J9J2*B!J9J7	J9I I0,I3 -J9/I00I3 3
J/='J*$J9*J//	J9c           	       #    X l         [        R                  " U R                  5       5      nU R                  (       a  U R                  R                  5          U H  n[        [        R                  " 5       5      nU R                  (       a  U R                  R                  Xe5        U R                  R                  SXVS[        R                  " 5       445      I S h  vN   M     / nSn[        U5      n	X:  Ga  UR                  5       (       a2  UR                  5       n
U
(       a  U H  nUR!                  5         M     U
eU R"                  (       d  U R$                  [        U5      -
  nUS:  a   U R                  R'                  5       u  nu  pVp[        R                  " U R)                  XSXn5      5      nUR+                  U5        U R                  (       aA  [        R                  " 5       U-
  nU R                  R-                  UU[.        R0                  S9  US-  nUS:  a  M  U(       aj  [        R4                  " US[        R6                  S9I S h  vN u  nnU H)  nUI S h  vN nSUR8                  ;  d  M  US-  nU7v   M+     [;        U5      nO+[        R<                  " U R>                  S-  5      I S h  vN   U RA                  5       I S h  vN   X:  a  GM  UR!                  5         U R                  (       a  U R                  RC                  5         g g  GND! [        R2                   a     GM  f = f N N N Nn! UR!                  5         U R                  (       a  U R                  RC                  5         f f = f7f)Nr   r   r   r   r   r   r=   )"rT   r1   r   r   rR   r   rJ   r   r   r   ry   r   r-   r   r   r   r   rz   rp   r   ra   r   r   r   r   r   r   r   r   r   r2   ro   r   r   )r   re   rT   r\   r   r#   r_   r   completed_count
total_urlsr   r   r   r   r   r   r   r8   r   r   r   r   s                         r   run_urls_stream(MemoryAdaptiveDispatcher.run_urls_stream  s      !,,T-F-F-HI<<LL L	$djjl+<<LL))'7oo))1sQ		.L*MNNN  LOTJ!.!&&(((224C!-AHHJ ".!	00 33c,6GGE!)"RVRaRaRlRlRnOH&Os[ $+#6#6 $sG Q$D )//5  $||,0IIK,,F	 $ 8 8$+.7+6+B+B !9 !" "QJE)  !)6  *1,,$cw?V?V+ %MD' +/'5!5 &V-A-AA+q0O"(L +/ $(=L "--(;(;a(?@@@ 33555s ".z !!#||!!# I OL  '11 "!"%
 "6 A 6 !!#||!!# s   AN
BM	 L"BM	 3B.L% !M	 ).M	 MM	 )M*M	  A M	  MM	 M	M	 %=N
"M	 %L>9M	 =L>>M	 M	 M	 M	 	>NN
)r|   ro   rT   rm   r{   rq   rp   rz   rl   rr   rn   rw   ry   )	g     V@g     W@g     @U@rB           @r   NN)r   )rD   rE   rF   rG   rH   rI   r   r   r   r    r   r   rJ   r   r	   r   r   ra   r   rg   r   r   r   rL   __classcell__r}   s   @r   rj   rj      s    +/,0,0 #"$"'/4.2,0="'= %*= %*	=
 =  =  = &e_= {+= .)= =4(5T	U 	 	 	  P
P
 &-=(>>?P
 	P

 P
 
P
d_3i_ !_ &-=(>>?	_
 
	 _B9,vZ$3iZ$ !Z$ &-=(>>?	Z$
 
)4/	0Z$ Z$r"   rj   c                      ^  \ rS rSr    SS\S\S\\   S\\   4U 4S jjjr SS\	S\
\\\   4   S	\	S
\R                  S\4
S jjrS\S\\	   S\
\\\   4   S\\   4S jrSrU =r$ )SemaphoreDispatcherio  semaphore_countrp   rQ   rR   c                 <   > [         TU ]  X45        Xl        X l        g r&   )ru   r    r   rp   )r   r   rp   rQ   rR   r}   s        r   r    SemaphoreDispatcher.__init__p  s     	/."4r"   r#   r\   r_   	semaphorer$   c                   #    [         R                   " 5       nSnS=pxU R                  X5      n	U	co  SU 3nU R                  (       a)  U R                  R                  U[        R
                  US9  [        UU[        USSS0SUS9S	S	U[         R                   " 5       US
9$  U R                  (       a)  U R                  R                  U[        R                  US9  U R                  (       a#  U R                  R                  U5      I S h  vN   U IS h  vN   [        R                  " 5       n
U
R                  5       R                  S-  nU R                  R!                  XUS9I S h  vN nU
R                  5       R                  S-  nX-
  =pxU R                  (       Ga  UR"                  (       a  U R                  R%                  XR"                  5      (       d  S['        U5      R(                   3nU R                  (       a(  U R                  R                  U[        R
                  S9  [        UUUUUU[         R                   " 5       US
9sS S S 5      IS h  vN   [         R                   " 5       nU R                  (       a  U R                  R                  UUUUUS9  $ $ UR*                  (       dF  UR,                  nU R                  (       a(  U R                  R                  U[        R
                  S9  O9U R                  (       a(  U R                  R                  U[        R.                  S9  S S S 5      IS h  vN   [         R                   " 5       nU R                  (       a  U R                  R                  UUUUUS9  [        UUWUUUUUS
9$  GN GN GN6 GNF Nc! , IS h  vN  (       d  f       Nx= f! [0         ad  n[3        U5      nU R                  (       a(  U R                  R                  U[        R
                  S9  [        US0 S[3        U5      S9n S nANS nAff = f! [         R                   " 5       nU R                  (       a  U R                  R                  UUUUUS9  f f = f7f)Nr   rt   r   r   r   r   Fr   r   )r_   r#   r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   )r-   r]   rR   r   r   r   r   r
   r   rQ   r9   r   r   r   r   rT   r   r;   r@   r   r'   r   r   r   r   rJ   )r   r#   r\   r_   r   r   r   r   r   r   r   r   r   r   r   r   s                   r   ra   SemaphoreDispatcher.crawl_url{  s     YY[
%(( ,,S9 "GuMM||((&--"/ )  %"&(9:!"/ %+ "9	||((K$;$;
 )    ''66s;;; y ..*&22488KH#||00Y`0aa$00266+F
-7-FF$$$););,,99#?Q?QRR*UV^_bVcVjVjUk(l<< LL44W[EWEW4X0$+ ##))5(3'1%)YY[*7	  !yyP yy{H||((%!- +"/ )  # ~~$*$8$8M||00ASAS0T\\LL,,W[=R=R,S; !yP yy{H||((%!- +"/ )  !%#!'	
 		
i <  b !yyy>  	FM||((9K9K(L b2uCPQFF		 yy{H||((%!- +"/ )  s   BQ;!A)O 
N
O NO AN')N*C&N'O N"O !AQ;&BN'6O N%O AQ;O O N'"O %O 'N>-N0.N>:O =P2 >O 
P/AP*%P2 *P//P2 2AQ88Q;rT   re   c           	        #    Xl         U R                  (       a  U R                  R                  5          [        R                  " U R
                  5      n/ nU H  n[        [        R                  " 5       5      nU R                  (       a  U R                  R                  Xv5        [        R                  " U R                  XcXt5      5      nUR                  U5        M     [        R                  " USS06I S h  vN U R                  (       a  U R                  R                  5         $ $  N0! U R                  (       a  U R                  R                  5         f f = f7f)Nreturn_exceptionsT)rT   rR   r   r1   	Semaphorer   rJ   r   r   r   r   ra   r   gatherr   )	r   rT   re   r\   r   tasksr#   r_   r   s	            r   rg   SemaphoreDispatcher.run_urls  s      <<LL 	$))$*>*>?IEdjjl+<<LL))'7**NN3C T"  !G$GG||!!#  H||!!# s/   2ECD- :D+;D- >-E+D- -.EE)rT   rp   r   )   r   NNr&   )rD   rE   rF   rG   rI   r   r   r   r    rJ   r   r	   r   r1   r  r   ra   r   rg   rL   r   r   s   @r   r   r   o  s      !"$.2,0	5	5  	5 {+		5
 .)	5 	5  (,m
m
 &-=(>>?m
 	m

 $$m
 
m
^$ $ 3i$ &-=(>>?	$
 
	 $ $r"   r   )#typingr   r   r   r   r   async_configsr	   modelsr
   r   r   r   components.crawler_monitorr   typesr   collections.abcr   r-   r   r1   r   urllib.parser   r3   abcr   r   utilsr   r   rO   rj   r   rM   r"   r   <module>r     sp    5 5 +  7 " *     !  # 09 9z8S 8vX$~ X$vU$. U$r"   