
    Mi"                        S SK JrJrJrJrJrJrJr  S SKrS SK	r	S SK
Jr  S SKrSSKJrJr  SSKJr  SSKJrJr  SSKJr   " S	 S
\5      r " S S\5      r " S S\5      r " S S5      rS r\S:X  a  \R<                  " \" 5       5        gg)    )ListOptionalUnionAsyncGeneratorDictAnyCallableN)urljoin   )BrowserConfigCrawlerRunConfig)CrawlResult)AsyncLoggerLogLevel)hooks_to_stringc                       \ rS rSrSrSrg)Crawl4aiClientError   z1Base exception for Crawl4ai Docker client errors. N__name__
__module____qualname____firstlineno____doc____static_attributes__r       p/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/docker_client.pyr   r          ;r   r   c                       \ rS rSrSrSrg)ConnectionError   z2Raised when connection to the Docker server fails.r   Nr   r   r   r   r!   r!      s    <r   r!   c                       \ rS rSrSrSrg)RequestError   z1Raised when the server returns an error response.r   Nr   r   r   r   r$   r$      r   r   r$   c                      \ rS rSrSr     S S\S\S\S\S\\   4
S	 jjr	S
\SS4S jr
S!S jr    S"S\\   S\\   S\\   S\\\\\4   \\\4   4      S\S\\\4   4S jjrS\S\S\R,                  4S jr    S"S\\   S\\   S\\   S\\\\\4   \\\4   4      S\S\\\\   \\S4   4   4S jjrS\\\4   4S jrS!S jrS#S jrS\\   S\\   S\\   SS4S jr Sr!g)$Crawl4aiDockerClient   zMClient for interacting with Crawl4AI Docker server with token authentication.Nbase_urltimeout
verify_sslverboselog_filec                     UR                  S5      U l        X l        [        U[        R
                  US9U l        [        R                  " UUSS0S9U l	        S U l
        g )N/)r-   	log_levelr,   zContent-Typeapplication/json)r*   verifyheaders)rstripr)   r*   r   r   DEBUGloggerhttpxAsyncClient_http_client_token)selfr)   r*   r+   r,   r-   s         r   __init__Crawl4aiDockerClient.__init__   sY     !,!8x~~W^_!--#%78

 &*r   emailreturnc                 R  #    [        U R                  S5      n U R                  R                  SU 3SS9  U R                  R                  USU0S9I Sh  vN nUR                  5         UR                  5       nUS   U l        S	U R                   3U R                  R                  S
'   U R                  R                  SSS9  g No! [        R                  [        R                  4 a8  nS[        U5       3nU R                  R                  USS9  [!        U5      eSnAff = f7f)z1Authenticate with the server and store the token.z/tokenzAuthenticating with email: AUTHtagr>   jsonNaccess_tokenBearer AuthorizationzAuthentication successfulzAuthentication failed: ERROR)r
   r)   r6   infor9   postraise_for_statusrE   r:   r3   successr7   r$   HTTPStatusErrorstrerrorr!   )r;   r>   urlresponsedatae	error_msgs          r   authenticate!Crawl4aiDockerClient.authenticate1   s    dmmX.	-KK:5'BO!..33Cw>N3OOH%%'==?D~.DK;B4;;-9PD%%o6KK ;H P ""E$9$9: 	-1#a&:IKKiW5!),,	-s;   D'<C CA.C D'C $D$,3DD$$D'c                   #     U R                   R                  [        U R                  S5      5      I Sh  vN   U R                  R                  SU R                   3SS9  g N,! [        R                   aB  nU R                  R                  S[        U5       3SS9  [        S[        U5       35      eSnAff = f7f)	z6Check if server is reachable, raising an error if not.z/healthNzConnected to READYrB   zServer unreachable: rI   zCannot connect to server: )r9   getr
   r)   r6   rM   r7   r$   rP   rO   r!   )r;   rT   s     r   _check_server"Crawl4aiDockerClient._check_serverA   s     	I##''y(IJJJKK- ?WM K!! 	IKK 4SVH=7K!$>s1vh"GHH	Is8   B>2A% A#+A% "B>#A% %B;9=B66B;;B>urlsbrowser_configcrawler_confighookshooks_timeoutc                 `   U R                   (       a&  SU R                    3U R                  R                  S'   UU(       a  UR                  5       O0 U(       a  UR                  5       O0 S.nU(       a;  [	        S UR                  5        5       5      (       a  [        U5      nOUnUUS.US'   U$ )z"Prepare request data from configs.rG   rH   )r]   r^   r_   c              3   8   #    U  H  n[        U5      v   M     g 7fN)callable).0vs     r   	<genexpr>8Crawl4aiDockerClient._prepare_request.<locals>.<genexpr>_   s     718A;;s   )coder*   r`   )r:   r9   r3   dumpanyvaluesr   )r;   r]   r^   r_   r`   ra   request_data
hooks_codes           r   _prepare_request%Crawl4aiDockerClient._prepare_requestJ   s     ;;;B4;;-9PD%%o6 7En11327En1132
 7777,U3
 #
 #(%L!
 r   methodendpointc                   #    [        U R                  U5      n U R                  R                  " X40 UD6I Sh  vN nUR	                  5         U$  N! [
        R                   a  n[        S[        U5       35      eSnAf[
        R                   a  n[        S[        U5       35      eSnAf[
        R                   a  nSUR                  R                  R                  SS5      ;   a3  UR                  R                  5       R                  S[        U5      5      O
[        U5      n[        SUR                  R                   S	U 35      eSnAff = f7f)
z)Make an HTTP request with error handling.NzRequest timed out: zFailed to connect: r1   zcontent-type detailzServer error z: )r
   r)   r9   requestrL   r7   TimeoutExceptionr!   rO   r$   rN   rR   r3   rZ   rE   status_code)r;   rr   rs   kwargsrQ   rR   rT   rU   s           r   _requestCrawl4aiDockerClient._requestm   s)    dmmX.	V!..66vMfMMH%%'O N %% 	B!$7Ax"@AA!! 	B!$7Ax"@AA$$ 	V-1C1C1G1GXZ1[[ *..xQ@ V  qzz/E/E.FbTUU		VsQ   E A AA EA E'A>>EB,,EBEEEc                   ^ ^#    T R                  5       I Sh  vN   T R                  XX4U5      mU=(       a    UR                  nT R                  R	                  S[        U5       SU(       a  SOS 3SS9  U(       a  S[        [        S4   4UU 4S	 jjnU" 5       $ T R                  S
STUS9I Sh  vN nUR                  5       n	U	R                  SS5      (       d  [        SU	R                  SS5       35      eU	R                  S/ 5       V
s/ s H  n
[        S0 U
D6PM     nn
T R                  R                  S[        U5       S3SS9  [        U5      S:X  a  US   $ U$  GNL Ns  sn
f 7f)a3  
Execute a crawl operation.

Args:
    urls: List of URLs to crawl
    browser_config: Browser configuration
    crawler_config: Crawler configuration
    hooks: Optional hooks - can be either:
           - Dict[str, Callable]: Function objects that will be converted to strings
           - Dict[str, str]: Already stringified hook code
    hooks_timeout: Timeout in seconds for each hook execution (1-120)

Returns:
    Single CrawlResult, list of results, or async generator for streaming

Example with function hooks:
    >>> async def my_hook(page, context, **kwargs):
    ...     await page.set_viewport_size({"width": 1920, "height": 1080})
    ...     return page
    >>>
    >>> result = await client.crawl(
    ...     ["https://example.com"],
    ...     hooks={"on_page_context_created": my_hook}
    ... )
Nz	Crawling z URLs z(streaming)ru   CRAWLrB   r?   c            
       >#    TR                   R                  STR                   S3TS9 IS h  vN n U R                  5         U R	                  5         S h  vN nUR                  5       (       d  M   [        R                  " U5      nSU;   a/  TR                  R                  UR                  SS5      US   S9  Mk  TR                  R                  UR                  SS5      SUR                  S	S
5      S9  UR                  S5      S:X  a  M  [        S0 UD67v   M   N N
 S S S 5      IS h  vN    g ! , IS h  vN  (       d  f       g = f7f)NPOSTz/crawl/streamrD   rP   rQ   unknown)rQ   rP   Ttimingg        )rQ   rM   r   status	completedr   )r9   streamr)   rL   aiter_linesstriprE   loadsr6   error_statusrZ   
url_statusr   )rR   lineresultrS   r;   s      r   stream_results2Crawl4aiDockerClient.crawl.<locals>.stream_results   s$    ,,33Ft}}o]<[bf3ggks--/&.&:&:&< 
<d::<<%)ZZ%5F&&0 $ 8 8VZZy=Yaghoap 8 q ( KK22vzz%7S]ajpjtjtu}  @C  kD2  E%zz(3{B (&1&;F&; ; h
<&< hgggggso   -ED"E!D9D&D$D&D95B-D9"E$D&&D9'E2D53E9E?E EEr   z/crawl)rE   r*   rM   FzCrawl failed: msgzUnknown errorresultszCrawl completed with z resultsr   r   r   )r[   rp   r   r6   rJ   lenr   r   r{   rE   rZ   r$   rM   )r;   r]   r^   r_   r`   ra   is_streamingr   rR   result_datarr   rS   s   `           @r   crawlCrawl4aiDockerClient.crawl~   se    B   """$$T>R_`%?.*?*?9SYKv|mY[5\]cjk<.d9J*K < < "##vxdMZZmmoy%000W/XYZZ-8__Y-KL-K;##-KL3CL>JPWX \Q.wqz;G;? 	#0 [
 Ms6   E.E$BE.2E'3AE.E)%A E.'E.)E.c                 b   #    U R                  SS5      I Sh  vN nUR                  5       $  N7f)zRetrieve configuration schemas.GETz/schemaN)r{   rE   )r;   rR   s     r   
get_schemaCrawl4aiDockerClient.get_schema   s)     ui88}} 9s   /-/c                    #    U R                   R                  SSS9  U R                  R                  5       I Sh  vN   g N7f)zClose the HTTP client session.zClosing clientCLOSErB   N)r6   rJ   r9   acloser;   s    r   closeCrawl4aiDockerClient.close   s5     )w7&&(((s   8AA Ac                    #    U $ 7frd   r   r   s    r   
__aenter__Crawl4aiDockerClient.__aenter__   s
     s   exc_typeexc_valexc_tbc                 @   #    U R                  5       I S h  vN   g  N7frd   )r   )r;   r   r   r   s       r   	__aexit__Crawl4aiDockerClient.__aexit__   s     jjls   )r9   r:   r)   r6   r*   )zhttp://localhost:8000g      >@TTN)r?   N)NNN   )r?   r'   )"r   r   r   r   r   rO   floatboolr   r<   rV   r[   r   r   r   r   r   r	   intr   rp   r7   Responser{   r   r   r   r   r   r   type	Exceptionr   r   r   r   r   r'   r'      s   W 0"&** * 	*
 * 3-*$- - - I 3759FJ!3i! !/! !!12	!
 d3=14S>ABC! ! 
c3h!FVS VC Venn V( 3759FJ@<3i@< !/@< !!12	@<
 d3=14S>ABC@< @< 
{D-~k4>O/PP	Q@<D$sCx. 
)
 )AT ^fgj^k pt r   r'   c                  Z  #    [        SS9 IS h  vN n U R                  S5      I S h  vN   U R                  S/5      I S h  vN n[        U5        U R	                  5       I S h  vN n[        U5        S S S 5      IS h  vN   g  Nv N_ NG N& N! , IS h  vN  (       d  f       g = f7f)NT)r,   zuser@example.comzhttps://example.com)r'   rV   r   printr   )clientr   schemas      r   mainr      s}     #D11V!!"4555||%:$;<<f((**f 2115<*	 2111sx   B+BB+BB	BB"B&B'B6B+BB+	BBBB+B(BB($B+__main__)typingr   r   r   r   r   r   r	   r7   rE   urllib.parser
   asyncioasync_configsr   r   modelsr   async_loggerr   r   utilsr   r   r   r!   r$   r'   r   r   runr   r   r   <module>r      s{    M M M      :  / "	) 	
	) 	
	& 	
r rl zKK r   