
    Mig                        S SK r S SKrS SKJr  S SKrS SKrS SKJrJr  S SK	J
r
  S SKrSSKJrJrJr  S SKrSSKJr  SSKJrJr  SS	KJr  SS
KJrJr  \ R2                  R5                  \ R6                  " S\R8                  " 5       5      S5      =rr\ R>                  " \SS9  \ R2                  R5                  \S5      r " S S5      r \ " 5       r!g)    N)Path)OptionalDict)asynccontextmanager   )CrawlResultMarkdownGenerationResultStringCompatibleMarkdown)AsyncLogger)ensure_content_dirsgenerate_content_hash)VersionManager)get_error_contextcreate_box_messageCRAWL4_AI_BASE_DIRECTORY	.crawl4aiTexist_okzcrawl4ai.dbc            
       &   \ rS rSrS!S\S\4S jjrS rS r\S 5       r	S r
S	 rS
 rS\4S jrS\S\\   4S jrS\S\\   4S jr   S"S\S\\   S\\   S\\   4S jjrS\4S jrS\4S jrS rS rS\S\S\4S jrS\S\S\\   4S jrS rg)#AsyncDatabaseManager   	pool_sizemax_retriesc                    [         U l        [        [        R                  R                  [         5      5      U l        Xl        X l        0 U l	        [        R                  " 5       U l        [        R                  " 5       U l        [        R                  " U5      U l        SU l        [#        5       U l        ['        [        R                  R)                  [*        SS5      SSS9U l        g )NFr   zcrawler_db.log
   )log_fileverbose	tag_width)DB_PATHdb_pathr   ospathdirnamecontent_pathsr   r   connection_poolasyncioLock	pool_lock	init_lock	Semaphoreconnection_semaphore_initializedr   version_managerr   joinbase_directorylogger)selfr   r   s      q/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/async_database.py__init__AsyncDatabaseManager.__init__   s    01IJ"&@B  $+$5$5i$@!!-/!WW\\.+?OP
    c           	        #     U R                   R                  SSS9  [        R                  " [        R                  R                  U R                  5      SS9  U R                  R                  5       nU R                  5       I Sh  vN   [        R                  " U R                  SS9 ISh  vN nUR                  S	5       ISh  vN nUR                  5       I Sh  vN nU(       d  [        S
5      eSSS5      ISh  vN   SSS5      ISh  vN   U(       a|  U R                   R                  SSS9  U R                  5       I Sh  vN   SSKJn  U" 5       I Sh  vN   U R                  R%                  5         U R                   R'                  SSS9  gU R                   R'                  SSS9  g GN/ GN
 N N N! , ISh  vN  (       d  f       N= f N! , ISh  vN  (       d  f       N= f N N! [         aF  nU R                   R)                  SSS[+        U5      0S9  U R                   R                  SSS9  e SnAff = f7f)z+Initialize the database and connection poolzInitializing databaseINIT)tagTr   N      >@timeoutzISELECT name FROM sqlite_master WHERE type='table' AND name='crawled_data'z"crawled_data table was not createdz%New version detected, running updatesr   )run_migrationz%Version update completed successfullyCOMPLETEz.Database initialization completed successfullyz&Database initialization error: {error}ERRORerrormessager8   paramsz)Database will be initialized on first userA   r8   )r0   infor!   makedirsr"   r#   r    r-   needs_updateainit_db	aiosqliteconnectexecutefetchone	Exceptionupdate_db_schema
migrationsr<   update_versionsuccessr?   str)r1   rF   dbcursorresultr<   es          r2   
initializeAsyncDatabaseManager.initialize*   s    0	KK4&AKK5E  //<<>L --/!! !((tDD::_ #)??#44F!'(LMM  ED   !Hf U++--- $o%%$$335##; $  ##D* $ 3 " E 5    EDDD .
 &  
	KK@Q(  
 KKC   
	s  IBG< F6'G< /F90G< 3G	F<
GG!F>"G8GG GG< G8G< G8G< !G:"8G< IG< 5I6G< 9G< <G>G GG	G	G	GG< G5$G'%G51G< :G< <
IAIIIc                 <  #    U R                    ISh  vN   U R                  R                  5        H  nUR                  5       I Sh  vN   M     U R                  R	                  5         SSS5      ISh  vN   g Nh N5 N	! , ISh  vN  (       d  f       g= f7f)z&Cleanup connections when shutting downN)r(   r%   valuescloseclear)r1   conns     r2   cleanupAsyncDatabaseManager.cleanup^   s_     >>>,,335jjl"" 6  &&( ">>" ">>>sV   BA<B1BA>	"B+B6B 7B>B BBB	BBc                	  #    U R                   (       dV  U R                   ISh  vN   U R                   (       d    U R                  5       I Sh  vN   SU l         SSS5      ISh  vN   U R                  R                  5       I Sh  vN   [        [        R                  " 5       5      n U R                   ISh  vN   X@R                   ;  a   ["        R$                  " U R&                  S
S9I Sh  vN nUR)                  S5      I Sh  vN   UR)                  S5      I Sh  vN   UR)                  S5       ISh  vN nUR+                  5       I Sh  vN nU Vs/ s H  oS   PM	     n	n1 Skn
U
[-        U	5      -
  nU(       a  [/        SU 35      eSSS5      ISh  vN   XPR                   U'   SSS5      ISh  vN   U R                   U   7v    U R                   ISh  vN   X@R                   ;   a2  U R                   U   R1                  5       I Sh  vN   U R                   U	 SSS5      ISh  vN   U R                  R3                  5         g GN4 GN! [         aR  nSSKn[        UR                  5       5      nU R                  R                  SSS[        U5      US   US   S.S	9  e SnAff = f GNX! , ISh  vN  (       d  f       GNn= f GNT GN# GN GN GN GN GNs  snf  GNT! , ISh  vN  (       d  f       GNj= f! [         aq  nSSKn[        UR                  5       5      nSUS    SUS    SUS    S[        U5       SUS    3
nU R                  R                  SSS[        U5      0S/S9  e SnAff = f GN! , ISh  vN  (       d  f       GN= f! [         aq  nSSKn[        UR                  5       5      nSUS    SUS    SUS    S[        U5       SUS    3
nU R                  R                  SSS[        U5      0S/S9  e SnAff = f GNF GN GN! , ISh  vN  (       d  f       GN= f! U R                   ISh  vN    X@R                   ;   a3  U R                   U   R1                  5       I Sh  vN    U R                   U	 SSS5      ISh  vN    O! , ISh  vN  (       d  f       O= fU R                  R3                  5         f = f7f)z4Connection pool manager with enhanced error handlingNTr   zSDatabase initialization failed:
{error}

Context:
{context}

Traceback:
{traceback}r>   code_contextfull_traceback)r?   context	tracebackrA   r8   force_verboserB   r9   r:   zPRAGMA journal_mode = WALzPRAGMA busy_timeout = 5000PRAGMA table_info(crawled_data)r   >   urlhtmllinksmediarP   markdownmetadata
screenshotcleaned_htmldownloaded_filesresponse_headersextracted_contentzDatabase missing columns: z.Unexpected error in db get_connection at line line_noz in functionz (filenamez
):
Error: z

Code context:
z{error}r?   )rA   r8   rB   boxes)r,   r)   rV   rL   sysr   exc_infor0   r?   rQ   r+   acquireidr&   current_taskr(   r%   rH   rI   r    rJ   fetchallset
ValueErrorrZ   release)r1   rU   rv   error_contexttask_idr\   rS   columnscolcolumn_namesexpected_columnsmissing_columnserror_messages                r2   get_connection#AsyncDatabaseManager.get_connectione   sp       ~~~(("oo///,0)	 &~* ''//111W))+,N	0~~~"6"661%.%6%6t||T%RR"ll+FGGG"ll+GHHH $(<<=$ $#,2OO,=&=G>E+FgsFgL+F0, /?\AR.RO.&0&@@Q$R'" !"+$ $2 9=,,W5C &~j &&w//( ~~~222..w7==???,,W5 &~ %%--/M & 0$ "(9#,,.(I)) %A '*.),Q+8+H-:;K-L$	 * 	  &~~~* 	2 &  SGH$ '>+F	$ $ $ $4 % "(9#,,.(IL][dMeLf g""/
";!<B}Z?X>Y Z&&)!fX ...;N.K-LN & ))$- '$+S-?#@#*)	 *  #E &~~~n  	-clln=M@yAY@Z [#J/0=3L2M Na& """/"?!@B  KK!]!34i	   !	$ &? &~~~t~~~222..w7==???,,W5 &~~~~~ %%--/s\  "S4IS4K IIIS4%J=&!S4K"S4+N/ ;K<N/ ?N"L2K!3LK$L$K'%L>K*?LK8K-K8K0-)K8L!K5"L4N/ ?N N/ S4'P-(S4+0P6P0P6-S48P39 S4I
J:(AJ55J::K =S4 KK	K	S4N/ !L$L'L*L-K80K85L8L	>L?L	L
NA,N		NNN/ N,NN,'N/ /
P*9A,P%%P**Q -S40P63S46Q<P?=Q	S4S1"Q%
#S1'0R<R
R<*S15R86S1<SSS"S11S4c                 :  #    [        U R                  5       HY  n U R                  5        ISh  vN nU" U/UQ76 I Sh  vN nUR                  5       I Sh  vN   UsSSS5      ISh  vN   s  $    g NE N6 N  N! , ISh  vN  (       d  f       M~  = f! [         au  nX0R                  S-
  :X  a3  U R
                  R                  SSSU R                  [        U5      S.S9  e [        R                  " SUS-   -  5      I Sh  vN     SnAM  SnAff = f7f)z,Execute database operations with retry logicNr   z2Operation failed after {retries} attempts: {error}r>   T)retriesr?   rd   )
ranger   r   commitrL   r0   r?   rQ   r&   sleep)r1   	operationargsattemptrR   rT   rU   s          r2   execute_with_retry'AsyncDatabaseManager.execute_with_retry   s     T--.G7..00B#,R#7$#77F))+%%! 100 /07% 1000  	7..22KK%% T#&*+/+;+;c!fM	 &  mmA1$5666	7s   DBA6BA>A8A>A:A>B+A<,B0D6B8A>:A><B>B	BB	BDB
D#A$DD
DDDDc                 "  #    [         R                  " U R                  SS9 ISh  vN nUR                  S5      I Sh  vN   UR	                  5       I Sh  vN   SSS5      ISh  vN   g NF N/ N N! , ISh  vN  (       d  f       g= f7f)zInitialize database schemar9   r:   Na{  
                CREATE TABLE IF NOT EXISTS crawled_data (
                    url TEXT PRIMARY KEY,
                    html TEXT,
                    cleaned_html TEXT,
                    markdown TEXT,
                    extracted_content TEXT,
                    success BOOLEAN,
                    media TEXT DEFAULT "{}",
                    links TEXT DEFAULT "{}",
                    metadata TEXT DEFAULT "{}",
                    screenshot TEXT DEFAULT "",
                    response_headers TEXT DEFAULT "{}",
                    downloaded_files TEXT DEFAULT "{}"  -- New column added
                )
            )rH   rI   r    rJ   r   )r1   rR   s     r2   rG   AsyncDatabaseManager.ainit_db   so     $$T\\4@@B**  $ ))+' A@@$ ' A@@@sg   $BA-BA5A/ A5A1A5B'A3(B/A51A53B5B;A><BBc                   #    [         R                  " U R                  SS9 ISh  vN nUR                  S5      I Sh  vN nUR	                  5       I Sh  vN nU Vs/ s H  oDS   PM	     nn/ SQnU H#  nXE;  d  M
  U R                  XA5      I Sh  vN   M%     UR                  5       I Sh  vN   SSS5      ISh  vN   g N N Nrs  snf  N< N" N! , ISh  vN  (       d  f       g= f7f)z Update database schema if neededr9   r:   Nrf   r   )
rj   ri   rl   rm   rp   ro   etaglast_modifiedhead_fingerprint	cached_at)rH   rI   r    rJ   r{   aalter_db_add_columnr   )r1   rR   rS   r   columnr   new_columnss          r2   rM   %AsyncDatabaseManager.update_db_schema   s     $$T\\4@@B::&GHHF"OO--G4;<G&1IGL<K &-33F??? & ))+/ A@@H-<& @/ A@@@s   $C1CC1CC CC
C C.C CCC0C1C5C1 CC1C
CCCC1C.C C.*C1
new_columnc                   #    US:X  a  UR                  SU S35      I Sh  vN   OAUS:X  a  UR                  SU S35      I Sh  vN   OUR                  SU S35      I Sh  vN   U R                  R                  SS	S
U0S9  g Nd NB N&7f)zAdd new column to the databaserp   z$ALTER TABLE crawled_data ADD COLUMN z TEXT DEFAULT "{}"Nr   z REAL DEFAULT 0z TEXT DEFAULT ""z'Added column '{column}' to the databaser7   r   r@   )rJ   r0   rD   )r1   r   rR   s      r2   r   )AsyncDatabaseManager.aalter_db_add_column  s     ++**6zlBVW   ;&**6zl/R   **6zlBRS   	=j) 	 	

s3   BB#BBB#B
$#BB
Brg   returnc           
         ^ ^#    U U4S jn T R                  U5      I Sh  vN $  N! [         a1  nT R                  R                  SSSS[	        U5      0S9   SnAgSnAff = f7f)z'Retrieve cached URL data as CrawlResultc                 b  >#    U R                  ST45       IS h  vN nUR                  5       I S h  vN nU(       d   S S S 5      IS h  vN   g UR                   Vs/ s H  o3S   PM	     nn[        [	        XB5      5      nUS   US   US   US   US   US   S.nUR                  5        HK  u  pxU(       a;  TR                  UUR                  S	5      S   5      I S h  vN n	U	=(       d    S
XW'   MG  S
XW'   MM     / SQn
U
 H)  n XW   (       a  [        R                  " XW   5      O0 XW'   M+     [        US   [        5      (       a$  US   R                  S5      (       a  US   S   US'    US   (       a  [        R                  " US   5      O/ US'   [         R"                  R%                  5       nUR                  5        VVs0 s H  u  pX;   d  M  X_M     nnnUS   US'   [!        S0 UD6sS S S 5      IS h  vN   $  GN GN GNs  snf  GN/! [        R                   aD    US:X  a5  [        XW   [        5      (       a  [        XW   =(       d    S
S
S
S
S
S9XW'    GM_  0 XW'    GMg  f = f! [        R                   a    / US'    Nf = fs  snnf  N! , IS h  vN  (       d  f       g = f7f)Nz(SELECT * FROM crawled_data WHERE url = ?r   rh   rn   rk   rq   rm   )rh   rn   rk   rq   rm   screenshots_ )rj   ri   rl   rp   rk   )raw_markdownmarkdown_with_citationsreferences_markdownfit_markdownfit_htmlr   ro    )rJ   rK   descriptiondictzipitems_load_contentsplitjsonloadsJSONDecodeError
isinstancerQ   r	   r   getr   __annotations__keys)rR   rS   rowr   r   row_dictcontent_fieldsfield
hash_valuecontentjson_fieldsvalid_fieldskvfiltered_dictr1   rg   s                  r2   _get2AsyncDatabaseManager.aget_cached_url.<locals>._get.  s    zz:SF "OO--   >D=O=OP=Okq>=OPG 12 %V,$,^$< ( 4)12E)F"*<"8#+L#9" *8)=)=)?%E!(,(:(:&!KK,Q/) # +2-R*, *@ )E1;C?DJJx7PR ! )$ hz2D99
+//??/7
/CN/S,6 $$67 

8,>#?@ /0  +::??A2:..2B X2B$!aFW2B X,4Z,@j)"3]3Y   . Q #*  // 1 J.:hos3S3S.F-5_-B8:46-/)+/HO /1HO10 ++ 635H/06
 !YU   s   J/HJ/JHJ J/HJ/JH-A4J!H"$J%H,A J-(I.1JJJJ0J/<J=J/JJ/JAI+J"I+&J*I++J.J
J	J

	JJ/J,JJ,(J/Nz$Error retrieving cached URL: {error}r>   Tr?   rd   r   rL   r0   r?   rQ   )r1   rg   r   rU   s   ``  r2   aget_cached_url$AsyncDatabaseManager.aget_cached_url+  si     M	4^		006666 	KK>"Q(	   	s6   A&( &( A&( 
A#'AA&A##A&c           
         ^#    U4S jn U R                  U5      I Sh  vN $  N! [         a1  nU R                  R                  SSSS[	        U5      0S9   SnAgSnAff = f7f)z
Retrieve only cache validation metadata for a URL (lightweight query).

Returns dict with: url, etag, last_modified, head_fingerprint, cached_at, response_headers
This is used for cache validation without loading full content.
c                   >#    U R                  ST45       IS h  vN nUR                  5       I S h  vN nU(       d   S S S 5      IS h  vN   g UR                   Vs/ s H  o3S   PM	     nn[        [	        XB5      5      n US   (       a  [
        R                  " US   5      O0 US'   UsS S S 5      IS h  vN   $  N N Nws  snf ! [
        R                   a    0 US'    N9f = f N/! , IS h  vN  (       d  f       g = f7f)NzSELECT url, etag, last_modified, head_fingerprint, cached_at, response_headers
                   FROM crawled_data WHERE url = ?r   rp   )rJ   rK   r   r   r   r   r   r   )rR   rS   r   r   r   r   rg   s         r2   _get_metadata?AsyncDatabaseManager.aget_cache_metadata.<locals>._get_metadata  s    zz6  "OO--   >D=O=OP=Okq>=OPG 126 $$67 

8,>#?@=? /0  +  
 . Q ++ 635H/06%   s   DB?DC+CC+ DCDC+C-C+(C
,C+-D9C):DC+DC+
C&#C+%C&&C+)D+D1C42D>DNz(Error retrieving cache metadata: {error}r>   Tr?   rd   r   )r1   rg   r   rU   s    `  r2   aget_cache_metadata(AsyncDatabaseManager.aget_cache_metadata  sg     	 0		00???? 	KKB"Q(	   	s6   A$& $& A$& 
A!'AA$A!!A$Nr   r   r   c           
         ^^^^#    UUUU4S jn U R                  U5      I Sh  vN   g N! [         a1  nU R                  R                  SSSS[	        U5      0S9   SnAgSnAff = f7f)zw
Update only the cache validation metadata for a URL.
Used to update etag/last_modified after a successful validation.
c                   >#    / n/ nTb"  UR                  S5        UR                  T5        Tb"  UR                  S5        UR                  T5        Tb"  UR                  S5        UR                  T5        U(       d  g UR                  T5        U R                  SSR                  U5       S3[        U5      5      I S h  vN   g  N7f)Nzetag = ?zlast_modified = ?zhead_fingerprint = ?zUPDATE crawled_data SET z, z WHERE url = ?)appendrJ   r.   tuple)rR   updatesrY   r   r   r   rg   s      r2   _update<AsyncDatabaseManager.aupdate_cache_metadata.<locals>._update  s     GFz*d#(23m,+56./MM#***499W+=*>nMf  s   B>C	CC	Nz&Error updating cache metadata: {error}r>   Tr?   rd   r   )r1   rg   r   r   r   r   rU   s    ````  r2   aupdate_cache_metadata+AsyncDatabaseManager.aupdate_cache_metadata  sd     	 	.	))'222 	KK@"Q(	  	s6   
A+- +- A+- 
A('A#A+#A((A+rT   c           
        ^^
^^^^#    TR                   S4TR                  =(       d    SS4STR                  =(       d    SS4TR                  =(       d    SS4S.n [	        TR
                  [        5      (       a  TR
                  S4US'   O[	        TR
                  [        5      (       a   TR
                  R                  5       S4US'   Of[	        TR
                  [        5      (       a)  [        TR
                  S	9nUR                  5       S4US'   O[        5       R                  5       S4US'    0 mUR                  5        H$  u  nu  pgU R                  Xg5      I Sh  vN TU'   M&     TR                  =(       d    0 nUR!                  S5      =(       d    UR!                  S5      =(       d    SmUR!                  S5      =(       d    UR!                  S5      =(       d    Sm[#        TSS5      =(       d    Sm[$        R$                  " 5       m
U
UUUUU4S jn	 U R'                  U	5      I Sh  vN   g! [         aN  nU R                  R                  S
[        U5       3SS9  [        5       R                  5       S4US'    SnAGNWSnAff = f GN1 Nc! [         a1  nU R                  R)                  SSSS[        U5      0S9   SnAgSnAff = f7f)zCache CrawlResult datarh   r   cleanedN	extractedr   )rh   rn   rk   rq   rm   rk   )r   z#Error processing markdown content: WARNINGrC   r   ETagzlast-modifiedzLast-Modifiedr   c                   >#    U R                  STR                  TS   TS   TS   TS   TR                  [        R                  " TR
                  5      [        R                  " TR                  5      [        R                  " TR                  =(       d    0 5      TS   [        R                  " TR                  =(       d    0 5      [        R                  " TR                  =(       d    / 5      TTTT45      I S h  vN   g  N7f)Na  
                INSERT INTO crawled_data (
                    url, html, cleaned_html, markdown,
                    extracted_content, success, media, links, metadata,
                    screenshot, response_headers, downloaded_files,
                    etag, last_modified, head_fingerprint, cached_at
                )
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                ON CONFLICT(url) DO UPDATE SET
                    html = excluded.html,
                    cleaned_html = excluded.cleaned_html,
                    markdown = excluded.markdown,
                    extracted_content = excluded.extracted_content,
                    success = excluded.success,
                    media = excluded.media,
                    links = excluded.links,
                    metadata = excluded.metadata,
                    screenshot = excluded.screenshot,
                    response_headers = excluded.response_headers,
                    downloaded_files = excluded.downloaded_files,
                    etag = excluded.etag,
                    last_modified = excluded.last_modified,
                    head_fingerprint = excluded.head_fingerprint,
                    cached_at = excluded.cached_at
            rh   rn   rk   rq   rm   )
rJ   rg   rP   r   dumpsrj   ri   rl   rp   ro   )rR   r   content_hashesr   r   r   rT   s    r2   _cache/AsyncDatabaseManager.acache_url.<locals>._cache  s     **4 JJ"6*">2":."#67NNJJv||,JJv||,JJv4"5"<0JJv66<"=JJv66<"=!$!5, , ,s   C:D=D>DzError caching URL: {error}r>   Tr?   rd   )rh   rn   rq   rm   r   rk   r
   r	   model_dump_jsonrQ   rL   r0   warningr   _store_contentrp   r   getattrtimer   r?   )r1   rT   content_mapmarkdown_resultrU   r   r   content_typerp   r   r   r   r   r   r   s    `        @@@@@r2   
acache_urlAsyncDatabaseManager.acache_url  s     [[&)#006B	B"(":":"@b+!N!,,2MB
	&//+CDDOO+J' FOO-EFFOO335+J' FOOS11":"X#335+J' -.>>@+J' .9.?.?.A*E*G*.*=*=g*T$TN5! /B "228b##F+Q/?/C/CF/KQr(,,_=lAQAUAUVeAfljl"6+=tDJIIK	-	 -	^	))&111M  	KK=c!fXFI   
 )*::<'K
#	 %Ut 2 	KK4"Q(	  	s   AK0H7 K>H7 KAH7 KH7 2/K!J"B:KJ 1J2J 6K7
JAJ
K
JKJ 
K!'KKKKc           
         #    S n U R                  U5      I Sh  vN $  N! [         a1  nU R                  R                  SSSS[	        U5      0S9   SnAgSnAff = f7f)	zGet total number of cached URLsc                    #    U R                  S5       IS h  vN nUR                  5       I S h  vN nU(       a  US   OSsS S S 5      IS h  vN   $  N; N% N	! , IS h  vN  (       d  f       g = f7f)Nz!SELECT COUNT(*) FROM crawled_datar   )rJ   rK   )rR   rS   rT   s      r2   _count5AsyncDatabaseManager.aget_total_count.<locals>._countQ  sO     zz"EFF&%00$*vay GFF0 GFFFsT   A4AA4AAAA4AA4AA4A1 A#!A1-A4Nz"Error getting total count: {error}r>   Tr?   rd   r   r   )r1   r   rU   s      r2   aget_total_count%AsyncDatabaseManager.aget_total_countN  sf     	2
		008888 	KK<"Q(	   	s6   A "  " A " 
A'AA AA c           
         #    S n U R                  U5      I Sh  vN   g N! [         a1  nU R                  R                  SSSS[	        U5      0S9   SnAgSnAff = f7f)z Clear all data from the databasec                 B   #    U R                  S5      I S h  vN   g  N7f)NzDELETE FROM crawled_datarJ   rR   s    r2   _clear.AsyncDatabaseManager.aclear_db.<locals>._cleard  s     **7888   Nz Error clearing database: {error}r>   Tr?   rd   r   )r1   r   rU   s      r2   	aclear_dbAsyncDatabaseManager.aclear_dba  s^     	9	))&111 	KK:"Q(	  	6   A!# !# A!# 
A'AA!AA!c           
         #    S n U R                  U5      I Sh  vN   g N! [         a1  nU R                  R                  SSSS[	        U5      0S9   SnAgSnAff = f7f)zDrop the entire tablec                 B   #    U R                  S5      I S h  vN   g  N7f)Nz!DROP TABLE IF EXISTS crawled_datar   r   s    r2   _flush.AsyncDatabaseManager.aflush_db.<locals>._flusht  s     **@AAAr   Nz Error flushing database: {error}r>   Tr?   rd   r   )r1   r   rU   s      r2   	aflush_dbAsyncDatabaseManager.aflush_dbq  s_     	B	))&111 	KK:"Q(	  	r   r   r   c                   #    U(       d  g[        U5      n[        R                  R                  U R                  U   U5      n[        R                  R                  U5      (       dJ  [        R                  " USSS9 ISh  vN nUR                  U5      I Sh  vN   SSS5      ISh  vN   U$ U$  N1 N N! , ISh  vN  (       d  f       U$ = f7f)z+Store content in filesystem and return hashr   wutf-8encodingN)	r   r!   r"   r.   r$   existsaiofilesopenwrite)r1   r   r   content_hash	file_pathfs         r2   r   #AsyncDatabaseManager._store_content  s     ,W5GGLL!3!3L!A<P	 ww~~i((}}YgFF!ggg&&& GF | G& GFF sZ   A?CB3CB9B5B9C*B7+	C5B97C9C?C CCr  c                   #    U(       d  g[         R                  R                  U R                  U   U5      n [        R
                  " USSS9 ISh  vN nUR                  5       I Sh  vN sSSS5      ISh  vN   $  N- N N	! , ISh  vN  (       d  f       g= f!   U R                  R                  SSSSU0S	9   g= f7f)
z$Load content from filesystem by hashNrr  r  z#Failed to load content: {file_path}r>   Tr  rd   )	r!   r"   r.   r$   r	  r
  readr0   r?   )r1   r  r   r  r  s        r2   r   "AsyncDatabaseManager._load_content  s      GGLL!3!3L!A<P	
	}}YgFF!VVX~ GFF% GFFF	KK="#Y/	   s   6CB! BB! B+B,B/B! ;B<B!  CB! BB! BBBB! CB! ! CC)r,   r%   r+   r$   r    r)   r0   r   r(   r   r-   )r      )NNN)__name__
__module____qualname____firstlineno__intr3   rV   r]   r   r   r   rG   rM   rQ   r   r   r   r   r   r   r   r   r   r   r  r   r   __static_attributes__r   r5   r2   r   r      s)   
# 
 
"2h) i0 i0V7&.6
S 
*[ [+1F [z(S (Xd^ (Z #'+*.** sm*  }	*
 #3-*Xn{ n` &  C s s /2	#r5   r   )"r!   r   pathlibr   rH   r&   typingr   r   
contextlibr   r   modelsr   r	   r
   r	  async_loggerr   utilsr   r   r   r   r   r"   r.   getenvhomer/   r   rE   r   async_db_managerr   r5   r2   <module>r$     s    	     ! *  S S  % = ! 877<<II($))+6  Gd #
'',,~}
5J
 J
\ () r5   