
    MiQ                        S SK J r  SSKrSSKrSSKrSSKrSSKJr  SSKJ	r	J
r
  SSKrSSKrSSKJr  S SKJrJrJrJrJrJr  S SKJr  S S	K7  S S
KJr  S S	K7  S S	K7  S SKJr  S SKJrJrJ r   S SK!J"r"J#r#  S SK$J%r%J&r&  S SK'J(r(  S SK)J*r*J+r+  S SK,J-r-J.r.J/r/J0r0  S S	K17  S SK1J2r2J3r3J4r4  S SK5J6r6  S SK7J8r8J9r9J:r:J;r;J<r<J=r=J>r>  S SK?J@r@JArA  S SKBJCrC   " S S5      rDg)   )__version__    N)Path)OptionalList)asynccontextmanager)CrawlResultMarkdownGenerationResultDispatchResultScrapingResultCrawlResultContainerRunManyReturn)async_db_manager)*)IdentityChunking)NoExtractionStrategy)AsyncCrawlerStrategyAsyncPlaywrightCrawlerStrategyAsyncCrawlResponse)	CacheModeCacheContext)DefaultMarkdownGeneratorMarkdownGenerationStrategy)DeepCrawlDecorator)AsyncLoggerAsyncLoggerBase)BrowserConfigCrawlerRunConfigProxyConfigSeedingConfig)BaseDispatcherMemoryAdaptiveDispatcherRateLimiter)AsyncUrlSeeder)sanitize_input_encodeInvalidCSSSelectorErrorfast_format_htmlget_error_contextRobotsParserpreprocess_html_for_schemacompute_head_fingerprint)CacheValidatorCacheValidationResult)
is_blockedc                      \ rS rSrSr0 rSS\" \R                  " S\	R                  " 5       5      5      SS4S\S\S\S	\S
\4
S jjrS rS rS rS r\S 5       r S S\S\S\4S jjrS\S\S\S\S\S\S\S\4S jr  S!S\\   S\\\\\   4      S\\   S\4S jjr S S\\\\   4   S\\    S\\\   \!\\\\\!\\"4   4      4   4   4S jjr#Sr$g)"AsyncWebCrawler9   a  
Asynchronous web crawler with flexible caching capabilities.

There are two ways to use the crawler:

1. Using context manager (recommended for simple cases):
    ```python
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(url="https://example.com")
    ```

2. Using explicit lifecycle management (recommended for long-running applications):
    ```python
    crawler = AsyncWebCrawler()
    await crawler.start()

    # Use the crawler multiple times
    result1 = await crawler.arun(url="https://example.com")
    result2 = await crawler.arun(url="https://another.com")

    await crawler.close()
    ```

Attributes:
    browser_config (BrowserConfig): Configuration object for browser settings.
    crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages.
    logger (AsyncLogger): Logger instance for recording events and errors.
    crawl4ai_folder (str): Directory for storing cache.
    base_directory (str): Base directory for storing cache.
    ready (bool): Whether the crawler is ready for use.

Methods:
    start(): Start the crawler explicitly without using context manager.
    close(): Close the crawler explicitly without using context manager.
    arun(): Run the crawler for a single source: URL (web, local file, or raw HTML).
    awarmup(): Perform warmup sequence.
    arun_many(): Run the crawler for multiple sources.
    aprocess_html(): Process HTML content.

Typical Usage:
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(url="https://example.com")
        print(result.markdown)

    Using configuration:
    browser_config = BrowserConfig(browser_type="chromium", headless=True)
    async with AsyncWebCrawler(config=browser_config) as crawler:
        crawler_config = CrawlerRunConfig(
            cache_mode=CacheMode.BYPASS
        )
        result = await crawler.arun(url="https://example.com", config=crawler_config)
        print(result.markdown)
NCRAWL4_AI_BASE_DIRECTORYFcrawler_strategyconfigbase_directorythread_safeloggerc                 *   U=(       d
    [        5       nXpl        U=(       d>    [        [        R                  R                  USS5      U R                  R                  SS9U l        UR                  5        VV	s0 s H  u  pUS;   d  M  X_M     n
nn	U=(       d    [        SUU R                  S.U
D6U l
        U(       a  [        R                  " 5       OSU l        [        R                  R                  US5      U l        [        R                  " U R                  SS9  [        R                  " U R                   S	3SS9  [!        5       U l        S
U l        ['        U 5      U l        U R)                  U R*                  5      U l        SU l        gs  sn	nf )a  
Initialize the AsyncWebCrawler.

Args:
    crawler_strategy: Strategy for crawling web pages. Default AsyncPlaywrightCrawlerStrategy
    config: Configuration object for browser settings. Default BrowserConfig()
    base_directory: Base directory for storing cache
    thread_safe: Whether to use thread-safe operations
    **kwargs: Additional arguments for backwards compatibility
z	.crawl4aizcrawler.log
   )log_fileverbose	tag_width)browser_configr7   NT)exist_okz/cacheF )r   r=   r   ospathjoinr;   r7   itemsr   r3   asyncioLock_lockcrawl4ai_foldermakedirsr)   robots_parserreadyr   _deep_handlerarun
url_seeder)selfr3   r4   r5   r6   r7   kwargsr=   kvparamss              s/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/async_webcrawler.py__init__AsyncWebCrawler.__init__r   sQ   *  2=?,  
WW\\.+}M''//!
 $*<<> )>41Q ;( 6($!$> ) 0 !
4R 5
);;5
 5
 (3W\\^
  "ww||NKH
D((48
t++,F3dC *^
 05&&tyy1	483)s   9F	Fc                    #    U R                   R                  5       I Sh  vN   U R                  R                  S[         3SS9  SU l        U $  N.7f)z
Start the crawler explicitly without using context manager.
This is equivalent to using 'async with' but gives more control over the lifecycle.
Returns:
    AsyncWebCrawler: The initialized crawler instance
Nz	Crawl4AI INITtagT)r3   
__aenter__r7   infocrawl4ai_versionrJ   rN   s    rS   startAsyncWebCrawler.start   sP      ##..0009%5$67VD
 	1s   AA/Ac                 Z   #    U R                   R                  SSS5      I Sh  vN   g N7f)z
Close the crawler explicitly without using context manager.
This should be called when you're done with the crawler if you used start().

This method will:
1. Clean up browser resources
2. Close any open pages and contexts
N)r3   	__aexit__r]   s    rS   closeAsyncWebCrawler.close   s%      ##--dD$???s   !+)+c                 >   #    U R                  5       I S h  vN $  N7fN)r^   r]   s    rS   rZ   AsyncWebCrawler.__aenter__   s     ZZ\!!!s   c                 @   #    U R                  5       I S h  vN   g  N7fre   )rb   )rN   exc_typeexc_valexc_tbs       rS   ra   AsyncWebCrawler.__aexit__   s     jjls   c                   #    S7v   g7f)u   异步空上下文管理器Nr?   r]   s    rS   nullcontextAsyncWebCrawler.nullcontext   s
      	s   	urlreturnc                 l  #    U R                   (       d  U R                  5       I Sh  vN   U=(       d
    [        5       n[        U[        5      (       a  U(       d  [        S5      eU R                  =(       d    U R                  5        ISh  vN    UR                  U R                  l        UR                  c  [        R                  Ul
        [        XR                  S5      nSnSnSnSnSn	[        R                  " 5       n
UR!                  5       (       a  ["        R$                  " U5      I Sh  vN nU(       Ga  UR&                  (       Ga  ["        R(                  " U5      I Sh  vN nU(       Ga  [+        UR,                  S9 ISh  vN nUR/                  UUR1                  S5      UR1                  S5      UR1                  S5      S9I Sh  vN nSSS5      ISh  vN   WR2                  [4        R6                  :X  a  S	Ul        U R                  R;                  S
SSUR<                  0S9  UR>                  (       d  UR@                  (       a=  ["        RB                  " UUR>                  UR@                  URD                  S9I Sh  vN   OUR2                  [4        RF                  :X  a/  SUl        U R                  RI                  SSSUR<                  0S9  O8U R                  R;                  SSSUR<                  0S9  SnOU(       a  SUl        U(       a  [K        URL                  5      n[K        URN                  =(       d    S5      n	U	(       a  U	S:X  a  SOU	n	URP                  nURR                  nURP                  (       a	  U(       d  SnURR                  (       a	  U(       d  SnU R                  RU                  URV                  [Y        U5      [        R                  " 5       U
-
  SS9  U(       a  URZ                  (       a  UR\                  (       av  URZ                  R_                  UR\                  UR`                  S9I Sh  vN nU(       a8  U R                  R;                  SSUR\                  URb                  S.S9  Xl2        OVURZ                  Rg                  5       I Sh  vN nU(       a-  U R                  R;                  SSSURb                  0S9  Xl2        U(       a  W(       Gd:  SSK4J5n  U(       an  URl                  (       a]  U Rn                  Rq                  XRr                  Rt                  5      I Sh  vN (       d!  [w        USSSS S!S"0S#9sSSS5      ISh  vN   $ URy                  S$5      =(       d    URy                  S%5      nS&[{        US'S5      -   nUR}                  5       nURd                  nSnSnSnSS/ SSS(.n[        U5       GH  nU(       a    GOUS:  a2  UUS)'   U R                  RI                  S*S+UUR                  USS, US-.S9  [        U5       GH3  u  nnUS:  d  US:  a>  U R                  R;                  S.S+US&-   [        U5      U(       a  URb                  OS/S0.S9  UUl2        US1==   S&-  ss'    [        R                  " 5       nURt                  (       a%  U R                  R                  URt                  5        U R                  R                  XS29I Sh  vN n[K        URL                  5      nURP                  nUR                  nUR                  nU R                  RU                  URV                  [Y        U5      [        R                  " 5       U-
  SS9  U R                  " S[XU	UUUUR                  URy                  S$5      (       a  S3OSUR                  U" U5      R                  S4.
UD6I Sh  vN nUR                  UlK        URy                  S$5      =(       d    URy                  S%5      nUR                  =(       d    U(       a  SOUUlI        UR                  UlL        UR                  UlM        UR                  UlN        UUlG        UR                  UlP        UR                  UlQ        UR                  UlR        UR                  UlS        [Y        U5      UlT        [{        US5S5      UlU        S6Ul        U(       a  SnSnO[        UR                  U5      u  nnUS7   R                  U(       a  URb                  OSUR                  UU(       a  UOSS8.5        U(       d  U(       a  SOS/US9'   S3n  GM  GM6     GM     UUl2        [{        US=S5      n!U!(       Ga   U(       d  U(       d  USL =(       d,    [        UR                  URL                  =(       d    S5      S   n"U"(       a  U R                  RI                  S>S+S?USS, 0S9  S3US@'    U!" U5      I Sh  vN n#U#(       aw  [K        U#5      n$ U R                  " S[UU$U	USSUR                  S3UU" U5      R                  S4.
UD6I Sh  vN nS3UlT        SDUlK        [{        US5S5      UlU        S6Ul        SFUS9'   U(       ai  UR1                  S@5      (       dK  U(       dD  [        UR                  URL                  =(       d    S5      u  nnU(       a  SUlT        SHU 3UlZ        UUl[        O [w        USSSU(       a  SIU 3OSJSK9nUUl[        U(       ab  URL                  (       aQ  URL                  R                  5       R                  SL5      n'U'SM:w  a"  URL                  SU'SN-    n([        U(5      Ul_        U R                  RU                  URV                  U(       a  UR                  OS[        R                  " 5       U
-
  SOS9  UR                  5       (       a.  [Y        U5      (       d  ["        R                  " U5      I Sh  vN   [        U5      sSSS5      ISh  vN   $ U R                  RU                  URV                  S3[        R                  " 5       U
-
  SOS9  [Y        U5      UlT        [{        US5S5      UlU        URy                  S$5      =(       d    URy                  S%5      nUR                  =(       d    U(       a  SOUUlI        [        U5      sSSS5      ISh  vN   $  GN GNr GN GN GNy GN5 GN(! , ISh  vN  (       d  f       GN>= f G
N GN GN~ GN GN GN8 GN]! [         a  n US7   R                  U(       a  URb                  OSSS3[	        U 5      S8.5        U R                  R                  US:U(       a  URb                  OS/ S;U  3S+S<9  [	        U 5      n[        U5      S&::  a  US&::  a  e  Sn A GM   Sn A GM  Sn A ff = f GN GN! [         a@  n%U R                  RI                  SAS+SB[	        U%5      SSC 0S9  [w        UU$S3SDSE9n Sn%A%GNSn%A%ff = f! [         a)  n&U R                  R                  USGU& 3S+S<9   Sn&A&GNSn&A&ff = f GN_ GNG GN! [         a  n)[        [        R                  " 5       5      n*SPU*SQ    SRU*SS    STU*SU    SV[	        U)5       SWU*SX    3
n+U R                  R                  UU+SYS<9  [        [w        USSU+SZ95      s Sn)A)sSSS5      ISh  vN    $ Sn)A)ff = f! , ISh  vN  (       d  f       g= f7f)\a  
Runs the crawler for a single source: URL (web, local file, or raw HTML).

Migration Guide:
Old way (deprecated):
    result = await crawler.arun(
        url="https://example.com",
        word_count_threshold=200,
        screenshot=True,
        ...
    )

New way (recommended):
    config = CrawlerRunConfig(
        word_count_threshold=200,
        screenshot=True,
        ...
    )
    result = await crawler.arun(url="https://example.com", config=config)

Args:
    url: The URL to crawl (http://, https://, file://, or raw:)
    config: Configuration object controlling crawl behavior
    [other parameters maintained for backwards compatibility]

Returns:
    CrawlResult: The result of crawling and processing
Nz4Invalid URL, make sure the URL is a non-empty stringF)timeoutetaglast_modifiedhead_fingerprint)ro   stored_etagstored_last_modifiedstored_head_fingerprinthit_validatedzCache validated: {reason}CACHEreasonmessagerY   rR   )ro   rs   rt   ru   hit_fallbackz/Cache validation failed, using cached: {reason}zCache stale: {reason}hit z[]FETCHro   successtimingrY   )ttlz3Using sticky proxy session: {session_id} -> {proxy}PROXY)
session_idproxyzSwitch proxy: {proxy}r   r   )urlparsei  zAccess denied by robots.txtzX-Robots-StatuszBlocked by robots.txt)ro   htmlr   status_codeerror_messageresponse_headerszraw:zraw://r   max_retries)attemptsretriesproxies_usedfallback_fetch_usedresolved_byr   u=   Anti-bot retry {attempt}/{max_retries} for {url} — {reason}ANTIBOTP   )attemptr   ro   r{   z#Trying proxy {idx}/{total}: {proxy}direct)idxtotalr   r   r4   T)
ro   r   extracted_contentr4   screenshot_datapdf_datar;   is_raw_htmlredirected_urloriginal_schemer   missr   )r   r   blockedr{   r   zProxy z	 failed: )ro   errorrY   fallback_fetch_functionzAAll retries exhausted, invoking fallback_fetch_function for {url}ro   r   z7Fallback HTML processing failed ({err}), using raw HTMLerrd      )ro   r   r   r   fallback_fetchzFallback fetch failed: z Blocked by anti-bot protection: zAll proxies failed: zAll proxies failed)ro   r   r   r   r   z</head>   COMPLETEz'Unexpected error in _crawl_web at line line_noz in functionz (filenamez
):
Error: z

Code context:
code_contextERROR)ro   r   r   r   r?   )frJ   r^   r   
isinstancestr
ValueErrorrF   rm   r;   r7   
cache_moder   ENABLEDr   timeperf_countershould_readr   aget_cached_urlcheck_cache_freshnessaget_cache_metadatar,   cache_validation_timeoutvalidategetstatusr-   FRESHcache_statusr[   r{   new_etagnew_last_modifiedaupdate_cache_metadatanew_head_fingerprintr   warningr%   r   r   
screenshotpdf
url_statusdisplay_urlboolproxy_rotation_strategyproxy_session_idget_proxy_for_sessionproxy_session_ttlserverproxy_configget_next_proxyurllib.parser   check_robots_txtrI   	can_fetchr=   
user_agentr	   
startswithgetattr_get_proxy_listranger   	enumeratelenr3   update_user_agentcrawlr   js_execution_resultaprocess_htmlr   schemer   redirected_status_coder   downloaded_files
mhtml_datamhtmlssl_certificatenetwork_requestsconsole_messagesr   r   r.   append	Exceptionerror_statusr   crawl_statslowerfindr+   ru   should_write
acache_urlr   r(   sysexc_info),rN   ro   r4   rO   cache_contextasync_responsecached_resultr   r   r   
start_timecache_metadata	validator
validationr   
next_proxyr   _is_raw_url_max_attempts_proxy_list_original_proxy_config_block_reason_donecrawl_result_crawl_stats_attempt_p_idx_proxyt1r   
is_raw_url_blocked
_crawl_err_fallback_fn_needs_fallback_fallback_html_sanitized_html	_proc_err_fallback_errhead_end	head_htmleerror_contextr   s,                                               rS   rL   AsyncWebCrawler.arun   sl    F zz**,-+-#s##3FH H ::3!1!1!333p&,nn# $$,(1(9(9F% !-S2C2CU K 6:-1"&$(!!..0
 !,,..*:*J*J3*O$OM !V%A%A%A+;+O+OPS+T%TN%#1&:Y:Y#Z#Z^g/8/A/A$',:,>,>v,F5C5G5G5X8F8J8JK]8^	 0B 0 *J $[#Z &,,0E0K0KK9HM6 KK,,(C$+(0*2C2C'D -   *22j6R6R&6&M&M(+)3)<)<2<2N2N5?5T5T	'" !" !" (..2G2M2MM9GM6 KK//(Y$+(0*2C2C'D 0  !KK,,(?$+(0*2C2C'D - 
 -1M"16M. 01C1CDD(=%77=2)%
  14E4M . & '4&>&>O,00H(((,zz((,KK**)55 $T
#002Z?#	 +  f<<..8>8V8V8l8l"33 & 8 8 9m 9 3
 & KK,,(]$+282I2I-7->->(" -  3=/ 9?8V8V8e8e8g2g
% KK,,(?$+(/1B1B'C - 
 3=/ %D5 &"9"9%)%7%7%A%A!4!4!?!?&     $/$'%'(-,/.K$57N2"	$K 433h #&.."8"TCNN8<TK$%q(I$IM"("8"8":K-3-@-@*$&M!E#'L$%#$(*/4'+$L %*-$8 !#a<6>L3 KK//(g$-/7393E3E+.s8.;	(" 0 	 /8.DNFF%zX\ $ 0 0,Q(1/5z14[1ABHh,& !1 !" 39F/(494X*%)%6%6%8#)#4#4$($9$9$K$K(.(9(9%; 8<7L7L7R7R$' 8S 88 28 (=^=P=P'Q2@2K2K+9+B+B6D6X6X 3 $ 6 6(5(A(A,0J+/+<+<+>+C(/	 !7 !" 6:5G5G 6"(+6G+14C-5,2NN8;v8N8NTY3A3P3P4<SM4H4H6" '-6" 0" <J;U;U 8-0^^F-C-_s~~V^G_
>L>[>[>|hr`dx{ ;FTFkFk C@N@_@_ =@N@_@_ =CV @5C5N5N 2?M?]?] <@N@_@_ =@N@_@_ =7;Dz 4:A&,X\:] 7<B 9 $//4H46M>H(6(B(BD?J$;Hm !-^ < C C>DV]]$3A3M3M/7?GmR	E" !# (0MS'YaL$?,0E$) (0e /E# %9x +AF' $+63Ld#SL#E+(D0 `),*B*BLDUDUD[Y[\]^_ ( + KK//(k$-(-s3Bx'8 0 
 CGL)>?+"7CC7H1H#16KN6[O%*=A=O=O >*031@>O39<@594:NN8<;><DSM<P<P>* /5>* 8*8 <@L$8?BL$<>Efl\`>aL$;@FL$=BRL$? $+//0EFF{6@ , 8 8,:K:K:Qr7S3Hm'7< 4?_`m_n=o :3?0
 (3 #!#$)(,Ta,@*Pg{( 4@0 $(9(9#/#4#4#:#:#<#A#A)#L#r>(4(9(9-8a<(HI<TU^<_L9KK**)558D 4 4%#002Z?&	 +  %1133D<O<O.99,GGG/=Y 433^ KK**)55 $#002Z?&	 +  -1JM)/6d04M, "%!7!S3>>(;SJ3@3O3O3p\fTXloM0/>w 433  4* %P &U#Z* $[#Z#Z#Z"!"n3  3h E 4X280"b $- * ,^ < C C>DV]]$37/3.1*o	E" !# !% 8 8(+,2F6==PX2YYbcmbn*o(1 !9 !"
 14J $'{#3q#8]a=O$) >P#8!*P 2I8* ,5 %* )-(;(;4m09493y>$3;O3P )< )*
 8C031@488;	8*%*( $- " $ 8 8(+,CM?*S(1 !9 !""b HU 4z   1#,,. A >mI>V=W X'
34B}Z7P6Q R!!fX &&&3N&C%DF  ((' )  ,b%} [ 433z{ 433sq  %|4s<A%|4s?|4|B y=3t47y=+t,#y=ty=Attty=&t'B*y=t,G y=t/A!y=4t25By=t5y=%|41t82|47D$y=A'ut;
Cu!t>
"Fu%By=x>w+x>$:w1w.w1#,x>Fy=y4y=$|40y71|46B4y=*|46y:7|4?|4y=y=y=ty=t)	tt)	$	y=/y=2y=5y=8|4;u>u
w(	B
w#	y=#w(	(y=+x>.w11
x;;5x60x>6x;;x>>
y1y,&y=,y11y=7|4:|4=
|A3|:|;|?|4||4|||1 |#!|1-|4r   r   r   r   r;   c                   ^^#^$#    [        USS5      (       a  SSKJn	  [        USS5      =(       d    UR                  S5      =(       d    Un
U	" TU
5      n[	        UTSUUR                  S	5      UR                  S
5      UR                  S5      UR                  S5      S9$ Sm# UR                  SS5      (       d  UOSn[
        R                  " 5       nUR                  nUR                  (       d  U R                  Ul        UR                  R                  5       nUR                  SS5        UR                  UR                  5        VVs0 s H  u  nnUUR                  5       ;  d  M  UU_M!     snn5        UR                  " UT40 UD6nUc  [!        SU 35      e [)        U[*        5      (       a{  [-        UR                  SS5      5      m#UR                  S0 5      n[)        U[*        5      (       a  UR                  S/ 5      O/ nUR                  S0 5      nUR                  S0 5      nO[-        UR.                  5      m#[1        UR2                  S5      (       a  UR2                  R5                  5       OUR2                  n[)        U[*        5      (       a  UR                  S/ 5      O/ n[1        UR6                  S5      (       a  UR6                  R5                  5       OUR6                  nUR8                  n[;        TSSS9m$UR<                  =(       d
    [?        5       n[        USS5      nU4S jU#4S jU$4S jS .nT#n UR                  UU#4S! j5      nU" 5       nUR                  S5      =(       d    UR                  S5      =(       d    Un[B        RD                  " S'T[B        RF                  5      nU(       a  URI                  S5      nURK                  UUS(9nU R                  RM                  US[O        [
        R                  " 5       U-
  S)-  5      S)-  S*S+9  [Q        U5      (       Gd  URR                  (       Ga  [)        URR                  [T        5      (       Gd  [
        R                  " 5       nURR                  RV                  nUS,:X  aN  URX                  (       d=  U R                  RM                  U[Q        T5      [
        R                  " 5       U-
  S-S+9  S.nURZ                  TT$T#URX                  S/.R                  UURZ                  5      n US0;   a
  []        5       OUR^                  n!U!Ra                  U 5      n"[1        URR                  S15      (       a%  URR                  Rc                  UU"5      I Sh  vN nO4[d        Rf                  " URR                  Rh                  UU"5      I Sh  vN n[j        Rl                  " US2[$        SS39nU R                  RM                  U[Q        T5      [
        R                  " 5       U-
  S-S+9  URn                  (       a  [q        T#5      m#[	        UTT$T#UUUUUUUUSSS49$ s  snnf ! ["         a  n[!        [%        U5      5      eSnAf[&         a  n[!        SU S[%        U5       35      eSnAff = f! [&         a?  nU R                  (       a!  U R                  RA                  S"U S#U S$3S%S&9  T#n SnAGNySnAff = f GNa GN/7f)5a  
Process HTML content using the provided configuration.

Args:
    url: The URL being processed
    html: Raw HTML content
    extracted_content: Previously extracted content (if any)
    config: Configuration object controlling processing behavior
    screenshot_data: Screenshot data (if any)
    pdf_data: PDF data (if any)
    verbose: Whether to enable verbose logging
    **kwargs: Additional parameters for backwards compatibility

Returns:
    CrawlResult: Processed result containing extracted and formatted content
prefetchFr   )quick_extract_linksbase_urlNr   Tr   r   r   )ro   r   r   linksr   r   r   r   r   r   zRaw HTMLro   z:Process HTML, Failed to extract content from the website: z	, error: cleaned_htmlmediatablesr  metadata
model_dumpi  i )html_contenttext_thresholdmax_sizecontent_sourcec                     > T $ re   r?   )r   s   rS   <lambda>/AsyncWebCrawler.aprocess_html.<locals>.<lambda>"  s        c                     > T $ re   r?   r  s   rS   r$  r%  #  s    Lr&  c                     > T $ re   r?   )fit_htmls   rS   r$  r%  $  s    r&  )raw_htmlr  r*  c                     > T $ re   r?   r(  s   rS   r$  r%  +  s    S_r&  zError getting/processing 'z' for markdown source: z. Falling back to cleaned_html.MARKDOWN_SRCrX   z*<base\s[^>]*href\s*=\s*["\']([^"\']+)["\'])
input_htmlr  i  SCRAPEr   fit_markdownEXTRACTmarkdown)r2  r   r*  r  r0  )r   r  r*  rL      )indentdefaultensure_ascii)ro   r   r*  r  r2  r  r  r  r  r   r   r   r   r   )9r   utilsr  r   r	   r   r   scraping_strategyr7   __dict__copypopupdaterC   keysscrapr   r&   r   r   r   dictr%   r  hasattrr  r  r  r  r*   markdown_generatorr   r   research
IGNORECASEgroupgenerate_markdownr   intr   extraction_strategyr   input_formatr0  raw_markdownr   chunking_strategychunkrL   rD   	to_threadrunjsondumps	prettiifyr'   )%rN   ro   r   r   r4   r   r   r;   rO   r  effective_urlr  _urlr  r8  rR   rP   rQ   resultr  r  r  r  rA  selected_html_sourcehtml_source_selectormarkdown_input_htmlsource_lambdar  base_tag_matchmarkdown_resultcontent_formatcontentchunkingsectionsr  r*  s%     `                                @@rS   r   AsyncWebCrawler.aprocess_html  s}    8 6:u--2 $FJ=dL\A]dadM'm<E"JJ}5!',>!?%zz*:; &

+< =
 
  	$jj>>3JD""$B !' 8 8$+++/;;!( __))+FJJud#MMFLLN 5NDAqfkkm3  1a4N 5 6 &7%<%<T&%#&%F ~ PQTPUV   fd##0

>2.0LJJw+E0:5$0G0GUYYx,RFJJw+Ezz*b1H01D1DEL 29|1T1TFLL++-Z`ZfZfE0:5$0G0GUYYx,RF18|1T1TFLL++-Z`ZfZfEH-4QT`gh %%C)A)C 	  ''9;K^\ %0( 
 +	/0445IK_`M"//0 ::j)PVZZ8H-IPS#PRVXZXeXef%++A.H 00.! 1  	 	))+b0D89D@	 	 	
  &''***v99;OPP""$B#77DDN/8T8T&&  $T
#002R7%	 '  ", ,88$ , / < < c./">">?  "%II !"-- 
  ~~g.H v116::*0*D*D*I*I$PX*Y$Y! +2*;*;..22C+ %! !%

!!Su!
 KK""  $T
#002R7%	 #  +L9L %$&/
 	
g5 ' 	%SV$$ 	LSEQZ[^_`[aZbc 	p  	/{{##01E0FF]^_]``  A& $ 
 #/	/f %Z%s   B[*$B(Y Y
+Y
2-Y F&[*Z #H [*[$4[*8['9B[*Y 
ZY))Z6ZZ[*
[!"4[[*[!![*'[*urls
dispatcherc                   ^ ^^^^^^^#    T=(       d
    [        5       m[        T[        5      (       a  TS   OTm[        TSS5      (       a~  TR                  (       a  UU U4S jnU" 5       $ / nT HT  nT R                  UTS9I Sh  vN n[        U[        5      (       a  UR                  U5        MC  UR                  U5        MV     U$ TcL  [        T[        5      (       a  TS   OTm[        TSS5      n	[        TSS	5      n
[        [        XU
-   4S
SS9S9mS m[        T[        5      (       a'  T(       a  TS   R                  OSnT(       a  TS   OSmOTR                  nTmUU 4S jmU(       a  UUUU UU4S jnU" 5       $  TR                  T TTS9I Sh  vN nU Vs/ s H  nT" U5      PM     snT" 5       I Sh  vN   $  GNB N/s  snf  N! T" 5       I Sh  vN    f = f7f)a  
Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy.

Args:
urls: List of URLs to crawl
config: Configuration object(s) controlling crawl behavior. Can be:
    - Single CrawlerRunConfig: Used for all URLs
    - List[CrawlerRunConfig]: Configs with url_matcher for URL-specific settings
dispatcher: The dispatcher strategy instance to use. Defaults to MemoryAdaptiveDispatcher
[other parameters maintained for backwards compatibility]

Returns:
Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]:
    Either a list of all results or an async generator yielding results

Examples:

# Batch processing (default)
results = await crawler.arun_many(
    urls=["https://example1.com", "https://example2.com"],
    config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
)
for result in results:
    print(f"Processed {result.url}: {len(result.markdown)} chars")

# Streaming results
async for result in await crawler.arun_many(
    urls=["https://example1.com", "https://example2.com"],
    config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True),
):
    print(f"Processed {result.url}: {len(result.markdown)} chars")
r   deep_crawl_strategyNc                    >#    T HP  n TR                  U TS9I S h  vN n[        U[        5      (       a  U H  nU7v   M
     M@  U  S h  vN nU7v   M     g  N> N
 M\  7f)Nr   )rL   r   list)ro   rT  rprimary_cfgrN   r`  s      rS   _deep_crawl_stream5AsyncWebCrawler.arun_many.<locals>._deep_crawl_stream  s\     #'+yy[y'I!I%fd33%+&' &, ,2 (a&'  $!I
(6s3   A"A+A"AAAA"AA"r   
mean_delayg?	max_rangeg333333?g      N@   )
base_delay	max_delayr   )rate_limiterc                     [        U R                  S[        U R                  U R                  U R
                  U R                  U R                  U R                  S95      =(       d    U R                  $ )Ndispatch_result)task_idmemory_usagepeak_memoryr   end_timer   )	setattrrT  r   rr  rs  rt  r   ru  r   )task_results    rS   transform_result3AsyncWebCrawler.arun_many.<locals>.transform_result  si    &&%" + 3 3%0%=%=$/$;$;#.#9#9!,!5!5&1&?&? & %%r&  Fc                  6  >#    T (       a  T R                   (       ay  T R                  (       ag  T R                  (       aU  T R                  R                  T R                   5      I S h  vN   TR                  R                  SSST R                   0S9  g g g g g  N07f)Nz)Auto-released proxy session: {session_id}r   r   r|   )r   proxy_session_auto_releaser   release_sessionr7   r[   )primary_configrN   s   rS   maybe_release_session8AsyncWebCrawler.arun_many.<locals>.maybe_release_session'  s     //9966$<<LL"33     G(.*I*IJ ! 	 7 : 0 s   A#B&B'1Bc                    >#     TR                  TTTS9  S h  vN n T" U 5      7v   M   N
 T" 5       I S h  vN    g ! T" 5       I S h  vN    f = f7f)Ncrawlerr`  r4   )run_urls_stream)rw  r4   ra  r~  rN   rx  r`  s    rS   result_transformer5AsyncWebCrawler.arun_many.<locals>.result_transformer6  s_     2-7-G-G $4 .H . <k /{;;< . 0111/111sJ   AA  .,.A  .A  
A<A AAAAr  )r   r   re  r   streamrL   extendr   r"   r#   run_urls)rN   r`  r4   ra  rO   rh  all_resultsro   rT  rj  rk  r  r  _resultsresr~  rg  r}  rx  s   ````           @@@@rS   	arun_manyAsyncWebCrawler.arun_many  s    f -+- $.fd#;#;fQi; 5t<<!!( *++ C#'99S9#EEF!&$//#**62#**62   #"'1&$'?'?&)VK lC@J[#>I1( *,BC" !J	$ fd##)/VAY%%UF*0VAYdN]]F#N	 2 2 &''.!+!4!4TU[!4!\\9AB#(-B ,---[ FR ]B .+---sm   A<G)GC?G)G GG $G6G 8
G)GG)G G G)G&G" G&&G)domain_or_domainsc                   #    U R                   (       d#  [        U R                  U R                  S9U l         U(       a  UR                  " S
0 UD6O[
        R                  " U5      nUR                  =(       d    U R                  Ul        U R                   R                  (       d  U R                  U R                   l        UR                  b&  UR                  U R                   R                  l        O/U R                  R                  U R                   R                  l        [        U[        5      (       aA  U R                  R                  SSSU0S9  U R                   R                  UU5      I Sh  vN $ [        U[        [        45      (       aJ  U R                  R                  SSS[        U5      0S9  U R                   R!                  UU5      I Sh  vN $ [#        S	5      e Nt N7f)a  
Discovers, filters, and optionally validates URLs for a given domain(s)
using sitemaps and Common Crawl archives.

Args:
    domain_or_domains: A single domain string (e.g., "iana.org") or a list of domains.
    config: A SeedingConfig object to control the seeding process.
            Parameters passed directly via kwargs will override those in 'config'.
    **kwargs: Additional parameters (e.g., `source`, `live_check`, `extract_head`,
              `pattern`, `concurrency`, `hits_per_sec`, `force_refresh`, `verbose`)
              that will be used to construct or update the SeedingConfig.

Returns:
    If `extract_head` is False:
        - For a single domain: `List[str]` of discovered URLs.
        - For multiple domains: `Dict[str, List[str]]` mapping each domain to its URLs.
    If `extract_head` is True:
        - For a single domain: `List[Dict[str, Any]]` where each dict contains 'url'
          and 'head_data' (parsed <head> metadata).
        - For multiple domains: `Dict[str, List[Dict[str, Any]]]` mapping each domain
          to a list of URL data dictionaries.

Raises:
    ValueError: If `domain_or_domains` is not a string or a list of strings.
    Exception: Any underlying exceptions from AsyncUrlSeeder or network operations.

Example:
    >>> # Discover URLs from sitemap with live check for 'example.com'
    >>> result = await crawler.aseed_urls("example.com", source="sitemap", live_check=True, hits_per_sec=10)

    >>> # Discover URLs from Common Crawl, extract head data for 'example.com' and 'python.org'
    >>> multi_domain_result = await crawler.aseed_urls(
    >>>     ["example.com", "python.org"],
    >>>     source="cc", extract_head=True, concurrency=200, hits_per_sec=50
    >>> )
)r5   r7   Nz)Starting URL seeding for domain: {domain}SEEDdomainr|   z(Starting URL seeding for {count} domainscountz:`domain_or_domains` must be a string or a list of strings.r?   )rM   r$   rG   r7   cloner    from_kwargsr5   r;   r   r   r[   r`  re  tupler   	many_urlsr   )rN   r  r4   rO   seeding_configs        rS   
aseed_urlsAsyncWebCrawler.aseed_urlsI  s    V  -#33{{DO 4://}?X?XY_?` )7(E(E(]I]I]%%%%)[[DOO" !!--;-C-CDOO""*-1[[-@-@DOO""* '--KKC "34  
 --!   )D%=99KKB%6!78   22!  
 YZZ!s%   E1G,3G(4A$G,G*G,*G,)
rK   rF   rL   r=   rG   r3   r7   rJ   rI   rM   re   )NN)%__name__
__module____qualname____firstlineno____doc___domain_last_hitr   r@   getenvr   homer   r   r   r   rT   r^   rb   rZ   ra   r   rm   r   r   rL   r	   r   r   r   Unionr!   r  r    DictAnyr  __static_attributes__r?   r&  rS   r0   r0   9   s   4l  26 $!II0$))+>@!"&:9.:9 :9 	:9 :9  :9x
	@"   $(\\ !\
 
\|@
@
 @
 	@

 !@
 @
 @
 @
 
@
J MQ/3	W.3iW. /6F1GGHIW. ^,	W.$ 
%W.x +/Y[ d3i0Y[ 'Y[
 
tCy$sDsDcN/B)C$DDEE	FY[ Y[r&  r0   )Er   r\   r@   rB  r   r   pathlibr   typingr   r   rO  rD   
contextlibr   modelsr	   r
   r   r   r   r   async_databaser   rK  r   content_filter_strategyrH  r   async_crawler_strategyr   r   r   r   r   r   markdown_generation_strategyr   r   deep_crawlingr   async_loggerr   r   async_configsr   r   r   r    async_dispatcherr!   r"   r#   async_url_seederr$   r7  r%   r&   r'   r(   r)   r*   r+   cache_validatorr,   r-   antibot_detectorr.   r0   r?   r&  rS   <module>r     s    8 	 	 
   !   +  -   / & " 5 
 3 . 6 V V  S S ,   C (i[ i[r&  