
    Mi9                         S SK r S SKrS SKJr  S SKrS SKJr  S SKrS SKrS SK	r	S SK
J
r
  SSKJrJr  \" \R                  SS9r " S	 S
5      rS\S\4S jrSS\\   4S jjrS r\S:X  a  \" 5         gg)    N)Path)Optional)datetime   )AsyncLoggerLogLevelT)	log_levelverbosec                   b    \ rS rSrS\4S jrS\S\4S jrS\S\4S jrS\S	\S\4S
 jr	S r
Srg)DatabaseMigration   db_pathc                 v    Xl         U R                  [        R                  R	                  U5      5      U l        g N)r   _ensure_content_dirsospathdirnamecontent_paths)selfr   s     m/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/migrations.py__init__DatabaseMigration.__init__   s'    !66rwww7OP    	base_pathreturnc                     SSSSSS.n0 nUR                  5        H=  u  pE[        R                  R                  X5      n[        R                  " USS9  XcU'   M?     U$ )	Nhtml_contentcleaned_htmlmarkdown_contentextracted_contentscreenshots)htmlcleanedmarkdown	extractedr"   T)exist_ok)itemsr   r   joinmakedirs)r   r   dirsr   keyr   r   s          r   r   &DatabaseMigration._ensure_content_dirs   sa    "%*,(
  JJLLC77<<	3DKKt,!%# ) r   contentc                     [         R                  " 5       nUR                  UR                  5       5        UR	                  5       nU$ r   )xxhashxxh64updateencode	hexdigest)r   r.   xcontent_hashs       r   _generate_content_hash(DatabaseMigration._generate_content_hash'   s1    LLN	!"{{}r   content_typec                   #    U(       d  gU R                  U5      n[        R                  R                  U R                  U   U5      n[        R                  R                  U5      (       dJ  [        R                  " USSS9 IS h  vN nUR                  U5      I S h  vN   S S S 5      IS h  vN   U$ U$  N1 N N! , IS h  vN  (       d  f       U$ = f7f)N wzutf-8)encoding)	r7   r   r   r)   r   existsaiofilesopenwrite)r   r.   r9   r6   	file_pathfs         r   _store_content DatabaseMigration._store_content.   s     227;GGLL!3!3L!A<P	ww~~i((}}YgFF!ggg&&& GF | G& GFF sZ   BCB9CB? B;!B?%C0B=1	C;B?=C?CCCCc                 V  #    [         R                  SSS9   [        R                  " U R                  5       ISh  vN nUR                  S5       ISh  vN nUR                  5       I Sh  vN nSSS5      ISh  vN   SnW H  nUu  nnnn	n
nU R                  US5      I Sh  vN nU R                  US5      I Sh  vN nU R                  U	S	5      I Sh  vN nU R                  U
S
5      I Sh  vN nU R                  US5      I Sh  vN nUR                  SUUUUUU45      I Sh  vN   US-  nUS-  S:X  d  M  [         R                  SU S3SS9  M     UR                  5       I Sh  vN   [         R                  SU S3SS9  SSS5      ISh  vN   g GNf GNO GN: GN-! , ISh  vN  (       d  f       GNC= f GN! GN
 N N N N Nb N<! , ISh  vN  (       d  f       g= f! [         a'  n[         R                  SSS[        U5      0S9  UeSnAff = f7f)z/Migrate existing database to file-based storagezStarting database migration...INITtagNzqSELECT url, html, cleaned_html, markdown, 
                       extracted_content, screenshot FROM crawled_datar   r#   r$   r%   r&   r"   aE  
                        UPDATE crawled_data 
                        SET html = ?, 
                            cleaned_html = ?,
                            markdown = ?,
                            extracted_content = ?,
                            screenshot = ?
                        WHERE url = ?
                    r   d   z	Migrated z records...zMigration completed. z records processed.COMPLETEMigration failed: {error}ERRORerrormessagerI   params)loggerinfo	aiosqliteconnectr   executefetchallrD   commitsuccess	ExceptionrN   str)r   dbcursorrowsmigrated_countrowurlr#   r   r%   r!   
screenshot	html_hashcleaned_hashmarkdown_hashextracted_hashscreenshot_hashes                     r   migrate_database"DatabaseMigration.migrate_database;   sC     	4&AE	 ((66"::J  !'!22D	  "#C $ )" '+&9&9$&G GI)-)<)<\9)U#UL*.*=*=h
*S$SM+/+>+>);, &N -1,?,?"M- 'O
 ** &()*+  ( #a'N%+q0i/?{$KQWXY  \ iik!!+N+;;NO"  q 766 3	   $ !H#U$S&'
0 "o 7666z  	LL3Q(  
 G	sR  H)$G5 F"G5  GF%GF..F(/F.3G>F+?)G(G	)GGGGG6G7GGG0G1G/G4G5GG5 GG5 !H)"G5 %G(F.+G.G	4F75G		GGGGGGGG5 G2!G$"G2.G5 1H)2G5 5
H&?"H!!H&&H))r   r   N)__name__
__module____qualname____firstlineno__r[   r   dictr   r7   rD   ri   __static_attributes__ r   r   r   r      sV    Q Qc d c c C s s Jr   r   r   r   c           	        #    [         R                  R                  U 5      (       d  [        R	                  SSS9  g[
        R                  " 5       R                  S5      nU  SU 3n [        R                  " S5      I Sh  vN   [        R                  " X5        [        R	                  SU 3S	S9  U$  N3! [         a'  n[        R                  S
SS[        U5      0S9  UeSnAff = f7f)z"Create backup of existing databasez,No existing database found. Skipping backup.rG   rH   Nz%Y%m%d_%H%M%Sz.backup_r   zDatabase backup created at: rK   rL   rM   rN   rO   )r   r   r>   rR   rS   r   nowstrftimeasynciosleepshutilcopy2rZ   rN   r[   )r   	timestampbackup_pathrh   s       r   backup_databaser{      s     77>>'""BO ''8IIXi[1KmmA 	W*2;-@jQ 	  /WgsSTvEV 	 	
 s<   A%C+(B7 B52B7 4C+5B7 7
C("C##C((C+c                 p  #    U c4  [         R                  R                  [        R                  " 5       SS5      n [         R                  R                  U 5      (       d  [        R                  SSS9  g[        U 5      I Sh  vN nU(       d  g[        U 5      nUR                  5       I Sh  vN   g N0 N7f)zRun database migrationNz	.crawl4aizcrawl4ai.dbz/No existing database found. Skipping migration.rG   rH   )r   r   r)   r   homer>   rR   rS   r{   r   ri   )r   rz   	migrations      r   run_migrationr      s     '',,tyy{KG77>>'""E6R (00K!'*I

$
$
&&& 1
 's$   A?B6B2*B6,B4-B64B6c                      SSK n U R                  SS9nUR                  SSS9  UR                  5       n[        R
                  " [        UR                  5      5        g)zCLI entry point for migrationr   Nz/Migrate Crawl4AI database to file-based storage)descriptionz	--db-pathzCustom database path)help)argparseArgumentParseradd_argument
parse_argsru   runr   r   )r   parserargss      r   mainr      sV    $$E % F *@ADKKdll+,r   __main__r   )r   ru   pathlibr   rT   typingr   r0   r?   rw   r   async_loggerr   r   DEBUGrR   r   r[   r{   r   r   rk   rq   r   r   <module>r      s~    	         / 
x~~t	<r rj3 3 4'# '$
- zF r   