
    Mif                     ^   S SK r S SKJr  S SKrS SKJrJr  \ R                  R                  \ R                  " S\R                  " 5       5      S5      q\ R                  " [        SS9  \ R                  R                  [        S5      qS	 rS S\4S jjrS rS\S\\\\\\\\\\\4	      4S jr    S!S\S\S\S\S\S\S
\S\S\S\4S jjrS\4S jrS rS rS"S\S\4S jjr\S:X  aB  \ R                  R5                  [        5      (       a  \ R6                  " [        5        \" 5         gg)#    N)Path)OptionalTupleCRAWL4_AI_BASE_DIRECTORYz	.crawl4aiT)exist_okzcrawl4ai.dbc                      [         R                  " [        5      n U R                  5       nUR	                  S5        U R                  5         U R                  5         g )Na  
        CREATE TABLE IF NOT EXISTS crawled_data (
            url TEXT PRIMARY KEY,
            html TEXT,
            cleaned_html TEXT,
            markdown TEXT,
            extracted_content TEXT,
            success BOOLEAN,
            media TEXT DEFAULT "{}",
            links TEXT DEFAULT "{}",
            metadata TEXT DEFAULT "{}",
            screenshot TEXT DEFAULT ""
        )
    )sqlite3connectDB_PATHcursorexecutecommitclose)connr   s     r/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/legacy/database.pyinit_dbr      s@    ??7#D[[]F
NN	  	KKMJJL    media
new_columnc                 "   [        5          [        R                  " [        5      nUR	                  5       nUR                  SU  S35        UR                  5         UR                  5         g ! [         a  n[        SU 35         S nAg S nAff = f)Nz$ALTER TABLE crawled_data ADD COLUMN z TEXT DEFAULT ""z2Error altering database to add screenshot column: 
check_db_pathr	   r
   r   r   r   r   r   	Exceptionprint)r   r   r   es       r   alter_db_add_screenshotr   #   sx    O	Hw'2:,>NO	
 	

 HB1#FGGH   AA, ,
B6B		Bc                  0    [         (       d  [        S5      eg )Nz%Database path is not set or is empty.)r   
ValueError r   r   r   r   1   s    7@AA r   urlreturnc                     [        5          [        R                  " [        5      nUR	                  5       nUR                  SU 45        UR                  5       nUR                  5         U$ ! [         a  n[        SU 35         S nAg S nAff = f)NzSELECT url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot FROM crawled_data WHERE url = ?zError retrieving cached URL: 
r   r	   r
   r   r   r   fetchoner   r   r   )r!   r   r   resultr   s        r   get_cached_urlr'   6   s}     Ow' WF	
 "

 -aS12s   AA+ +
B5BBhtmlcleaned_htmlmarkdownextracted_contentsuccesslinksmetadata
screenshotc
                 0   [        5          [        R                  " [        5      n
U
R	                  5       nUR                  SU UUUUUUUUU	4
5        U
R                  5         U
R                  5         g ! [         a  n[        SU 35         S nAg S nAff = f)Na  
            INSERT INTO crawled_data (url, html, cleaned_html, markdown, extracted_content, success, media, links, metadata, screenshot)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(url) DO UPDATE SET
                html = excluded.html,
                cleaned_html = excluded.cleaned_html,
                markdown = excluded.markdown,
                extracted_content = excluded.extracted_content,
                success = excluded.success,
                media = excluded.media,      
                links = excluded.links,    
                metadata = excluded.metadata,      
                screenshot = excluded.screenshot
        zError caching URL: r   )r!   r(   r)   r*   r+   r,   r   r-   r.   r/   r   r   r   s                r   	cache_urlr1   I   s     O")w' !	
8 	

 )#A3'(()s   A&A3 3
B=BBc                  "   [        5          [        R                  " [        5      n U R	                  5       nUR                  S5        UR                  5       nU R                  5         US   $ ! [         a  n[        SU 35         S nAgS nAff = f)Nz!SELECT COUNT(*) FROM crawled_datar   zError getting total count: r$   )r   r   r&   r   s       r   get_total_countr3   {   st    O	w':;"

ay +A3/0r   c                     [        5          [        R                  " [        5      n U R	                  5       nUR                  S5        U R                  5         U R                  5         g ! [         a  n[        SU 35         S nAg S nAff = f)NzDELETE FROM crawled_datazError clearing database: r   r   r   r   s      r   clear_dbr6      sg    O/w'12

 /)!-../   AA( (
B
2BB
c                     [        5          [        R                  " [        5      n U R	                  5       nUR                  S5        U R                  5         U R                  5         g ! [         a  n[        SU 35         S nAg S nAff = f)NzDROP TABLE crawled_datazError flushing database: r   r5   s      r   flush_dbr9      sg    O/w'01

 /)!-../r7   default_valuec                 (   [        5          [        R                  " [        5      nUR	                  5       nUR                  SU  SU S35        UR                  5         UR                  5         g ! [         a  n[        SU 35         S nAg S nAff = f)NzUPDATE crawled_data SET z = "z" WHERE screenshot IS NULLz!Error updating existing records: r   )r   r:   r   r   r   s        r   update_existing_recordsr<      s{    O	7w'&zl$}oE_`	
 	

 71!5667s   A"A/ /
B9BB__main__)r   ){}r>   r>    )r   r>   )ospathlibr   r	   typingr   r   pathjoingetenvhomer   makedirsr   strr   r   boolr'   r1   intr3   r6   r9   r<   __name__existsremover    r   r   <module>rN      st   	   "
'',,ryy!;TYY[I;
W Gd #
'',,w
.0H HB
	eCc3S#tS@AB4 /)	/)
/) /) 	/)
 /) /) /) /) /) /)d 	/	/7 7c 7 z	ww~~g
		'I	 r   