
    MiF                    V   S r SSKJrJr  SSKJrJrJrJrJ	r	J
r
Jr  SSKJrJr  SSKrSSKrSSKrSSKrSSKrSSKJrJr  SSKrSSKJr  SSKJr  SS	KJrJr  SS
KJ r J!r!  \ " S S5      5       r"\ " S S5      5       r# " S S\5      r$ " S S\$5      r% " S S\$5      r& " S S5      r'g)a  
Adaptive Web Crawler for Crawl4AI

This module implements adaptive information foraging for efficient web crawling.
It determines when sufficient information has been gathered to answer a query,
avoiding unnecessary crawls while ensuring comprehensive coverage.
    )ABCabstractmethod)DictListOptionalSetTupleAnyUnion)	dataclassfieldN)defaultdictCounter)Path)AsyncWebCrawler)CrawlerRunConfigLinkPreviewConfig)LinkCrawlResultc                   R   \ rS rSr% Sr\" \S9r\\	   \
S'   \" \S9r\\   \
S'   \" \S9r\\   \
S'   Sr\	\
S'   \" \S9r\\	\4   \
S	'   \" S
 S9r\\	\4   \
S'   \" S S9r\\	\4   \
S'   \" S S9r\\	\\   4   \
S'   Sr\\
S'   \" \S9r\\   \
S'   \" \S9r\\	   \
S'   Sr\\   \
S'   Sr \\   \
S'   \" \S9r!\\	   \
S'   Sr"\\   \
S'   \" \S9r#\\$\\   \4      \
S'   Sr%\	\
S'   S\&\	\'4   4S jr(\)S\&\	\'4   SS 4S j5       r*\+S\S\4S  j5       r,\+S!\4S" j5       r-S#r.g)$
CrawlState   z-Tracks the current state of adaptive crawling)default_factorycrawled_urlsknowledge_basepending_links querymetricsc                       [        [        5      $ Nr   int     x/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/adaptive_crawler copy.py<lambda>CrawlState.<lambda>$   s
    [QTEUr%   term_frequenciesc                       [        [        5      $ r!   r"   r$   r%   r&   r'   r(   %   s
    UXIYr%   document_frequenciesc                       [        [        5      $ r!   )r   setr$   r%   r&   r'   r(   &   s
    kZ]N^r%   documents_with_termsr   total_documentsnew_terms_historycrawl_orderNkb_embeddingsquery_embeddingsexpanded_queriescoverage_shapesemantic_gapsembedding_modelpathc           
         [        U5      nUR                  R                  SSS9  0 S[        U R                  5      _SU R
                   Vs/ s H  o R                  U5      PM     sn_SU R                   Vs/ s H  o3R                  5       PM     sn_SU R                  _SU R                  _S[        U R                  5      _S	[        U R                  5      _S
U R                  R                  5        VVs0 s H  u  pEU[        U5      _M     snn_SU R                   _SU R"                  _SU R$                  _SU R&                  b  U R&                  R)                  5       OS_SU R*                  b  U R*                  R)                  5       OS_SU R,                  _SU R.                  _SU R0                  _n[3        US5       n[4        R6                  " XgSS9  SSS5        gs  snf s  snf s  snnf ! , (       d  f       g= f)z"Save state to disk for persistenceTparentsexist_okr   r   r   r   r   r)   r+   r.   r/   r0   r1   r2   Nr3   r4   r6   r7   w   )indent)r   parentmkdirlistr   r   _crawl_result_to_dictr   
model_dumpr   r   dictr)   r+   r.   itemsr/   r0   r1   r2   tolistr3   r4   r6   r7   openjsondump)selfr8   crlinkkv
state_dictfs           r&   saveCrawlState.save5   s   Dz$6
D!2!23
H[H[\H["99"=H[\
 D<N<NO<NDoo/<NO
 TZZ	

 t||
 T%:%: ;
 #D)B)B$C
 #D<U<U<[<[<]$^<]DAQQZ<]$^
 t33
  !7!7
 4++
 D<N<N<ZT//668`d
 $BWBWBc 5 5 < < >im
  5 5
  T//!
" t33#

( $_IIjA. _% ]O
 %_ _s   G"8G'8G,
G22
H returnc                    [        U5      n[        US5       n[        R                  " U5      nSSS5        U " 5       n[	        WS   5      Ul        US    Vs/ s H  oPR                  U5      PM     snUl        US    Vs/ s H  n[        S0 UD6PM     snUl	        US   Ul
        US   Ul        [        [        US   5      Ul        [        [        US	   5      Ul        [        [        US
   R!                  5        VVs0 s H  u  pxU[	        U5      _M     snn5      Ul        US   Ul        US   Ul        US   Ul        SSKn	UR-                  S5      b  U	R/                  US   5      OSUl        UR-                  S5      b  U	R/                  US   5      OSUl        UR-                  S/ 5      Ul        UR-                  S/ 5      Ul        UR-                  SS5      Ul        U$ ! , (       d  f       GN= fs  snf s  snf s  snnf )zLoad state from diskrNr   r   r   r   r   r)   r+   r.   r/   r0   r1   r   r2   r3   r4   r6   r7   r   r$   )r   rH   rI   loadr-   r   _dict_to_crawl_resultr   r   r   r   r   r   r#   r)   r+   rF   r.   r/   r0   r1   numpygetarrayr2   r3   r4   r6   r7   )
clsr8   rQ   rP   stated	link_dictrN   rO   nps
             r&   rW   CrawlState.loadR   s    Dz$_1J   N!;<FPQaFbcFb 9 9! <FbcBL_B]^B]Yt0i0B]^ )"9-!,S*=O2P!Q%0jAW6X%Y"%0ZXnMoMuMuMw6xMwTQq#a&yMw6x%y" *+< =",-@"A&}5 	GQ~~VeGfGrbhhz/'BCx|MW^^\nMoM{*5G*H!I  BF!+0BB!G(nn_bA */@" E1 _
  d^
 7ys   G*G<HH*
G9rL   c                 r   Sn[        U S5      (       aX  U R                  (       aG  [        U R                  S5      (       a  U R                  R                  nO[        U R                  5      nU R                  U[        U S5      (       a  U R
                  O0 [        U S5      (       a  U R                  S.$ 0 S.$ )z(Convert CrawlResult to serializable dictr   markdownraw_markdownlinksmetadataurlcontentre   rf   )hasattrrc   rd   strrh   re   rf   )rL   markdown_contents     r&   rC    CrawlState._crawl_result_to_dictp   s     2z""r{{r{{N33#%;;#;#; #&r{{#3  66'!(W!5!5RXX2'.r:'>'>	
 	
 EG	
 	
r%   r^   c           	         ^  " S S5      m " U4S jS5      nU" U S   U R                  SS5      U R                  S0 5      U R                  S	0 5      S
9$ )z Convert dict back to CrawlResultc                       \ rS rSrS rSrg)6CrawlState._dict_to_crawl_result.<locals>.MockMarkdown   c                     Xl         g r!   rd   rK   ri   s     r&   __init__?CrawlState._dict_to_crawl_result.<locals>.MockMarkdown.__init__       $+!r%   rs   N__name__
__module____qualname____firstlineno__ru   __static_attributes__r$   r%   r&   MockMarkdownrp          ,r%   r~   c                   "   > \ rS rSrU 4S jrSrg)9CrawlState._dict_to_crawl_result.<locals>.MockCrawlResult   c                 D   > Xl         T" U5      U l        X0l        X@l        g r!   )rh   rc   re   rf   )rK   rh   ri   re   rf   r~   s        r&   ru   BCrawlState._dict_to_crawl_result.<locals>.MockCrawlResult.__init__   s     ,W 5"
 (r%   )re   rc   rf   rh   Nrx   r~   s   r&   MockCrawlResultr      s    ) )r%   r   rh   ri   r   re   rf   rg   )rZ   )r^   r   r~   s     @r&   rX    CrawlState._dict_to_crawl_result   sW    	, 	,	) 	) %EE)R(%%$UU:r*	
 	
r%   r$   )/ry   rz   r{   r|   __doc__r   r-   r   r   rk   __annotations__rB   r   r   r   r   r   r   rE   r   r   floatr)   r#   r+   r.   r/   r0   r1   r2   r   r
   r3   r4   r5   r6   r	   r7   r   r   rR   classmethodrW   staticmethodrC   rX   r}   r$   r%   r&   r   r      s   7"37L#c(7(-d(CND%C %d ;M4:;E3O %d ;GT#u*; (-=U'Vd38nV+0AY+Z$sCx.Z05F^0_$sCH}-_OS $)#>tCy>"48Kc8 $(M8C='&*hsm*"'"=d3i=$(NHSM(5:45PM4d5k5012POS/sDy) /: c4i( \  : 
+ 
$ 
 
" 
 
 
r%   r   c                      \ rS rSr% SrSr\\S'   Sr\	\S'   Sr
\	\S'   S	r\	\S
'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\   \S'   S r\\S!'   Sr\\   \S"'   S#r\	\S$'   S%r\\S&'   Sr \\S''   Sr!\\S('   S)r"\\S*'   Sr#\\S+'   Sr$\\S,'   S%r%\\S-'   Sr&\\S.'   Sr'\\S/'   Sr(\\S0'   S1r)\\S2'   S3r*\\S4'   S5 r+S6r,g)7AdaptiveConfig   z#Configuration for adaptive crawlingffffff?confidence_threshold   	max_depth   	max_pages   top_k_links皙?min_gain_thresholdstatisticalstrategy皙?saturation_thresholdconsistency_threshold皙?coverage_weight333333?consistency_weightsaturation_weight      ?relevance_weightnovelty_weight皙?authority_weightF
save_stateN
state_path&sentence-transformers/all-MiniLM-L6-v2r7   embedding_llm_config
   n_query_variations333333?coverage_thresholdalpha_shape_alphaembedding_coverage_radiusg      @embedding_k_expembedding_nearest_weightembedding_top_k_weightembedding_overlap_threshold"embedding_min_relative_improvementembedding_validation_min_score embedding_quality_min_confidenceffffff? embedding_quality_max_confidence-?embedding_quality_scale_factorc                 J   SU R                   s=::  a  S::  d   S5       e   S5       eU R                  S:  d   S5       eU R                  S:  d   S5       eU R                  S:  d   S5       eSU R                  s=::  a  S::  d   S5       e   S5       eU R
                  U R                  -   U R                  -   n[        US-
  5      S	:  d
   S
U 35       eU R                  U R                  -   U R                  -   n[        US-
  5      S	:  d
   SU 35       eSU R                  s=:  a  S:  d   S5       e   S5       eU R                  S:  d   S5       eSU R                  s=::  a  S::  d   S5       e   S5       eSU R                  s=::  a  S::  d   S5       e   S5       e[        U R                  U R                  -   S-
  5      S	:  d   S5       eSU R                   s=::  a  S::  d   S5       e   S5       eSU R"                  s=:  a  S:  d   S5       e   S5       eSU R$                  s=::  a  S::  d   S5       e   S5       eSU R&                  s=::  a  S::  d   S5       e   S5       eSU R(                  s=::  a  S::  d   S5       e   S5       eU R*                  S:  d   S5       eg)z!Validate configuration parametersr      z,confidence_threshold must be between 0 and 1zmax_depth must be positivezmax_pages must be positiveztop_k_links must be positivez*min_gain_threshold must be between 0 and 1      ?gMbP?z$Coverage weights must sum to 1, got z(Link scoring weights must sum to 1, got z1embedding_coverage_radius must be between 0 and 1z embedding_k_exp must be positivez0embedding_nearest_weight must be between 0 and 1z.embedding_top_k_weight must be between 0 and 1zEmbedding weights must sum to 1z3embedding_overlap_threshold must be between 0 and 1z:embedding_min_relative_improvement must be between 0 and 1z6embedding_validation_min_score must be between 0 and 1z8embedding_quality_min_confidence must be between 0 and 1z8embedding_quality_max_confidence must be between 0 and 1z/embedding_quality_scale_factor must be positiveN)r   r   r   r   r   r   r   r   absr   r   r   r   r   r   r   r   r   r   r   r   r   )rK   
weight_sums     r&   validateAdaptiveConfig.validate   s   D--22b4bb2b4bb2~~!?#??!~~!?#??!!#C%CC#D++0q0^2^^0^2^^0 ))D,C,CCdF\F\\
:#$u,a0TU_T`.aa,**T-@-@@4CXCXX
:#$u,e0XYcXd.ee, 4115A5j7jj5j7jj5##a'K)KK'D116Q6j8jj6j8jj6D//414f6ff4f6ff440043N3NNQTTUX]]  	A  `A  	A]D4499p;pp9p;pp94::>Q>|@||>|@||>D77<1<v>vv<v>vv<D99>Q>z@zz>z@zz>D99>Q>z@zz>z@zz>22Q6i8ii6r%   r$   )-ry   rz   r{   r|   r   r   r   r   r   r#   r   r   r   r   rk   r   r   r   r   r   r   r   r   r   boolr   r   r7   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r}   r$   r%   r&   r   r      su   -"%%%IsIsK ##!Hc! #&%%#&5& OU  ##"u" "e!NE!e! J $J$ DOSC+/(4./   $$"u" (+u* !OU  '*e)$'E'
 *.-
 14&3 -0"E/
 /2$e1.2$e2,1"E1jr%   r   c            
           \ rS rSrSr\S\S\4S j5       r\S\S\	S\
\\\4      4S j5       r\S\S\	S\4S j5       r\S\S	\
\   SS
4S j5       rSrg
)CrawlStrategy   z+Abstract base class for crawling strategiesr]   rT   c                    #    g7f)z@Calculate overall confidence that we have sufficient informationNr$   )rK   r]   s     r&   calculate_confidence"CrawlStrategy.calculate_confidence   
      	   configc                    #    g7f)z/Rank pending links by expected information gainNr$   rK   r]   r   s      r&   
rank_linksCrawlStrategy.rank_links  r   r   c                    #    g7f)!Determine if crawling should stopNr$   r   s      r&   should_stopCrawlStrategy.should_stop  r   r   new_resultsNc                    #    g7f)#Update state with new crawl resultsNr$   )rK   r]   r   s      r&   update_stateCrawlStrategy.update_state  r   r   r$   )ry   rz   r{   r|   r   r   r   r   r   r   r   r	   r   r   r   r   r   r   r}   r$   r%   r&   r   r      s    5
 u   j . TRWX\^cXcRdMe   z > d   
 kAR W[  r%   r   c            	       ,   \ rS rSrSrS rS\S\4S jrS\S\4S jr	S\S\4S jr
S\S\4S	 jrS\S
\S\\\\4      4S jrS\S\S\4S jrS\S\S\4S jrS\S\4S jrS\S
\S\4S jrS\S\\   SS4S jrS\S\\   4S jrS\S\\   4S jrSrg)StatisticalStrategyi  z1Pure statistical approach - no LLM, no embeddingsc                 .    0 U l         SU l        SU l        g )Ng333333?g      ?)	idf_cachebm25_k1bm25_brK   s    r&   ru   StatisticalStrategy.__init__  s    r%   r]   rT   c                   #    UR                   (       d  gU R                  U5      nU R                  U5      nU R                  U5      nX!R                  S'   X1R                  S'   XAR                  S'   SU-  SU-  -   SU-  -   nU$ 7f)z@Calculate confidence using coverage, consistency, and saturation        coverageconsistency
saturationr   r   )r   _calculate_coverage_calculate_consistency_calculate_saturationr   )rK   r]   r   r   r   
confidences         r&   r   (StatisticalStrategy.calculate_confidence  s     ##++E211%8//6
 %-j!'2m$&0l# 8^cK&77#
:JJ
s   BBc                    UR                   (       a  UR                  S:X  a  gU R                  UR                   R                  5       5      nU(       d  g/ nUR                  (       a#  [        UR                  R                  5       5      OSnU H  nUR                  R                  US5      nUR                  R                  US5      nUS:  ag  XqR                  -  nUS:  a3  [        R                  " SU-   5      [        R                  " SU-   5      -  OSn	USSU	-  -   -  n
UR                  U
5        M  UR                  S5        M     [        U5      [        U5      -  n[        S[        R                  " U5      5      $ )zCoverage scoring - measures query term presence across knowledge base

Returns a score between 0 and 1, where:
- 0 means no query terms found
- 1 means excellent coverage of all query terms
r   r   r   r   r   )r   r/   	_tokenizelowerr)   maxvaluesrZ   r+   mathlogappendsumlenminsqrt)rK   r]   query_termsterm_scoresmax_tftermtfdfdoc_coveragefreq_signal
term_scorer   s               r&   r   'StatisticalStrategy._calculate_coverage-  sF    {{e33q8nnU[[%6%6%899>9O9OU++2245UVD''++D!4B++//a8BAv!$9$99 JPRSdhhq2v.!f*1EEYZ *Q{1B-BC
"":.""3'  $ {#c+&66 3		(+,,r%   c                 6   [        UR                  5      S:  a  g/ n[        [        UR                  5      5       H  n[        US-   [        UR                  5      5       H  n[        U R	                  UR                  U   5      5      n[        U R	                  UR                  U   5      5      nU(       d  MZ  U(       d  Mc  [        XV-  5      [        XV-  5      -  nUR                  U5        M     M     U(       a  [        U5      [        U5      -  nU$ SnU$ )zQInformation overlap between pages - high overlap suggests coherent topic coverager>   r   r   r   )r   r   ranger-   _get_document_termsr   r   )	rK   r]   overlapsijterms_iterms_joverlapr   s	            r&   r   *StatisticalStrategy._calculate_consistencyV  s    u##$q( s5//01A1q5#e&:&:";<d66u7K7KA7NOPd66u7K7KA7NOP7ww!'"34s7;L7MMGOOG, = 2 h-#h-7K  Kr%   c                 $   UR                   (       d  g[        UR                   5      S:  a  gUR                   S   S:  a  UR                   S   OSnUR                   S   S:  a  UR                   S   OSnSX#-  -
  n[        S[        US5      5      $ )zIDiminishing returns indicator - are we still discovering new information?r   r>   r   r   r   )r0   r   r   r   )rK   r]   recent_rateinitial_rater   s        r&   r   )StatisticalStrategy._calculate_saturationq  s    &&u&&'!+ 6;5L5LR5PST5Te--b1Z[5:5L5LQ5ORS5Su..q1YZ +45
3J,--r%   r   c                 \  #    / nUR                    H  nUR                  UR                  ;   a  M  U R                  XA5      nU R	                  XA5      nSnUR
                  U-  UR                  U-  -   UR                  U-  -   nUR                  XH45        M     UR                  S SS9  U$ 7f)z'Rank links by expected information gainr   c                     U S   $ Nr   r$   xs    r&   r'   0StatisticalStrategy.rank_links.<locals>.<lambda>  s    !r%   Tkeyreverse)
r   hrefr   _calculate_relevance_calculate_noveltyr   r   r   r   sort)	rK   r]   r   scored_linksrM   	relevancenovelty	authorityscores	            r&   r   StatisticalStrategy.rank_links  s     ''DyyE... 11$>I--d:GI ,,y8))G34++i78E .! (& 	nd;s   B*B,rM   c                    UR                   (       a  U(       d  gSR                  [        SUR                  =(       d    SUR                  =(       d    SUR
                  (       a,  UR
                  R                  S0 5      R                  SS5      OSUR
                  (       a,  UR
                  R                  S0 5      R                  SS5      OSUR
                  (       a,  UR
                  R                  S0 5      R                  SS5      OS/5      5      R                  5       nU(       d  gUR                  (       a  UR                  S	:  a  UR                  $ [        U R                  UR                   R                  5       5      5      n[        U R                  U5      5      nU(       d  g[        XE-  5      [        U5      -  nU$ )
z3BM25 relevance score between link preview and queryr    Nr   metatitledescriptionkeywordsr   )r   joinfiltertextr)  	head_datarZ   r   contextual_scorer-   r   r   )rK   rM   r]   	link_textr   
link_termsr  s          r&   r  (StatisticalStrategy._calculate_relevance  sP   {{$ HHVDIIOJJ"?C~~DNNvr*..w;SUEI^^DNNvr*..}bAY[BF..DNNvr*..z2>VX+
   EG 	    T%:%:Q%>((( $..):):)<=>	23
k./#k2BBr%   c                    UR                   (       d  gSR                  [        SUR                  =(       d    SUR                  =(       d    SUR
                  (       a  UR
                  R                  SS5      OSUR
                  (       a  UR
                  R                  SS5      OSUR
                  (       a  UR
                  R                  SS5      OS/5      5      R                  5       n[        U R                  U5      5      nU(       d  g[        UR                  R                  5       5      nXE-
  nU(       a  [        U5      [        U5      -  nU$ S	nU$ )
z9Estimate how much new information this link might provider   r'  Nr   r)  r*  r+  r   r   )r   r,  r-  r.  r)  r/  rZ   r   r-   r   r)   keysr   )rK   rM   r]   r1  r2  existing_terms	new_termsr"  s           r&   r  &StatisticalStrategy._calculate_novelty  s   ## HHVDIIOJJ"/3~~DNNw+259^^DNN}b126..DNNz2.b+
   EG 	 	23
 U3388:;/	6@#i.3z?2 GJr%   c                    SnUR                   (       d  gUR                   R                  5       nSU;   d  SU;   a  US-  nSU;   d  SU;   a  US-  nSU;   d  S	U;   a  US
-  nUR                  S5      (       a  US
-  nOUR                  S5      (       a  US-  nUR                  b  SU-  SUR                  -  -   n[	        US5      $ )zASimple authority score based on URL structure and link attributesr   r   z/docs/z/documentation/r   z/api/z/reference/z/guide/z
/tutorial/r   z.pdf)z.jpgz.pngz.gifr   r   r   )r  r   endswithintrinsic_scorer   )rK   rM   r$  rh   s       r&   _calculate_authority(StatisticalStrategy._calculate_authority  s    yyiioo s?/36SLEc>]c1SLE|s2SLE <<SLE\\233SLE +%K#(<(<"<<E5#r%   c                 &  #    UR                   R                  SS5      nX2R                  :  a  g[        UR                  5      UR
                  :  a  gUR                  (       d  gUR                   R                  SS5      UR                  :  a  gg7f)r   r   r   Tr   F)r   rZ   r   r   r   r   r   r   )rK   r]   r   r   s       r&   r   StatisticalStrategy.should_stop  s      ]]&&|S9
444 u!!"f&6&66 "" ==\3/63N3NNs   BBr   Nc                   #    U GHV  n[        UR                  5      n UR                  R                  nU R                  UR                  5       5      n[        5       nU H+  nUR                  U==   S-  ss'   UR                  U5        M-     UR                  n	U HL  nXR                  U   ;  d  M  UR                  U==   S-  ss'   UR                  U   R                  U	5        MN     [        UR                  5      n
X-
  nUR                  R                  U5        U=R                  S-  sl        UR                   R                  UR                  5        GMY     g! [         a    [        SUR                   S35        Sn GNTf = f7f)r   zWarning: CrawlResult z has no markdown contentr   r   N)r   r)   rc   rd   AttributeErrorprintrh   r   r   r-   addr/   r.   r+   r0   r   r1   )rK   r]   r   resultold_term_countri   termsterm_setr   doc_idnew_term_countr7  s               r&   r    StatisticalStrategy.update_state	  s]    !F !7!78N //66  NN7==?3E uH&&t,1,T" 
 **F 99$??..t494..t488@ ! !!7!78N&7I##**95 !!Q&! $$VZZ0] " " -fjj\9QRSs/   FE!A;F4B-F!%F
F	F

Fr.  c                     [         R                  " SSU5      nUR                  5       nU Vs/ s H  n[        U5      S:  d  M  UPM     nnU$ s  snf )z%Simple tokenization - can be enhancedz[^\w\s]r'  r>   )resubsplitr   )rK   r.  tokensts       r&   r   StatisticalStrategy._tokenize;  sK     vvj#t, $2Vs1vz!V2 3s   AAcrawl_resultc                 ~    UR                   R                  =(       d    SnU R                  UR                  5       5      $ )z!Extract terms from a crawl resultr   )rc   rd   r   r   )rK   rR  ri   s      r&   r  'StatisticalStrategy._get_document_termsF  s-    ''44:~~gmmo..r%   )r   r   r   )ry   rz   r{   r|   r   ru   r   r   r   r   r   r   r   r   r	   r   r   r  r  r<  r   r   r   r   rk   r   r  r}   r$   r%   r&   r   r     s4   ;

 u &'- '- '-RJ 5 6.: .% ."j . TRWX\^cXcRdMe 6 j U @t J 5 4 % :z > d *01
 01kAR 01W[ 01d	c 	d3i 	/ /S	 /r%   r   c                      \ rS rSrSrS$S\S\4S jjrS\\   S\	4S	 jr
S
\	S\	S\	4S jrS
\	S\	S\	4S jrS%S\S\S\	4S jjrS&S\	S\4S jjrS'S\S\\	   4S jjrS\	S
\	S\\\	\4      4S jrS\\   S\\\	\4      S\	S\\\\4      4S jrS\S\4S jrS\S\S\\\\4      4S jrS\S\4S jrS\S\S\4S jrS\S\4S  jrS\S!\\   SS4S" jrS#r g)(EmbeddingStrategyiL  z?Embedding-based adaptive crawling using semantic space coverageNr7   
llm_configc                     U=(       d    SU l         X l        0 U l        0 U l        SU l        S U l        S U l        S U l        SU l        g )Nr   Fr   )	r7   rW  _embedding_cache_link_embedding_cache_validation_passed_distance_matrix_cache_kb_embeddings_hash_validation_embeddings_cache_kb_similarity_threshold)rK   r7   rW  s      r&   ru   EmbeddingStrategy.__init__O  sO    .Z2Z$ "%'""' '+##' ,0)(,%r%   textsrT   c                    #    SSK Jn  S[        R                  " S5      S.nU" UUU R                  5      I Sh  vN $  N7f)z&Get embeddings using configured methodr   get_text_embeddingsopenai/text-embedding-3-smallOPENAI_API_KEYprovider	api_tokenN)utilsrd  osgetenvr7   )rK   ra  rd  r   s       r&   _get_embeddings!EmbeddingStrategy._get_embeddings\  sH     .7#34 
 )   
 
 	
 
s   7A >A r3   r2   c                    SSK nUb  [        U5      S:X  a  g[        UR                  5      S:X  a  UR                  SS5      n[        UR                  5      S:X  a  UR                  SS5      nXR                  R                  USSS9-  nX#R                  R                  USSS9-  nUR                  XER                  5      nSU-
  nU$ )z3Compute distance matrix using vectorized operationsr   Nr   r  Taxiskeepdims)rY   r   shapereshapelinalgnormdotT)rK   r3   r2   r`   
query_normkb_normsimilarity_matrixdistance_matrixs           r&   _compute_distance_matrix*EmbeddingStrategy._compute_distance_matrixi  s     C$6!$; %%&!+/772>}""#q()11!R8M &		7GaZ^(__
))..QQU."VV FF:yy9 //r%   c                     SSK nUb  [        U5      S:X  a  gUb  [        UR                  5       5      OSnU R                  b  X@R
                  :w  a  U R                  X5      U l        X@l        U R                  $ )z Get distance matrix with cachingr   N)rY   r   hashtobytesr\  r]  r}  )rK   r3   r2   r`   kb_hashs        r&   _get_cached_distance_matrix-EmbeddingStrategy._get_cached_distance_matrix  sv     C$6!$; 4A3L$},,./RV''////*.*G*GHX*hD''.$***r%   r   n_syntheticc                   #    SSK Jn  [        US-  5      nSU SU S3nU R                  (       a  U R                  R	                  SS5      OSnU R                  (       a  U R                  R	                  S	5      OS
nS/ SQ0nS/ SQ0nSS
Kn	US   R                  5       n
U	R                  U
5        [        S[        [        U
5      S-  5      5      nX* S
 nU/U
S
U*  -   nU R                  U5      I S
h  vN nXl        X4$  N7f)zJGenerate a point cloud representing the semantic neighborhood of the queryr   )perform_completion_with_backoffg?z	Generate z; variations of this query that explore different aspects: 'z'
        
        These should be queries a user might ask when looking for similar information.
        Include different phrasings, related concepts, and specific aspects.
        
        Return as a JSON array of strings.rh  zopenai/gpt-4o-miniri  Nqueries)z2what are the best vegetables to use in fried rice?z0how do I make vegetable fried rice from scratch?z8can you provide a quick recipe for vegetable fried rice?zMwhat cooking techniques are essential for perfect fried rice with vegetables?z*how to add flavor to vegetable fried rice?zAare there any tips for making healthy fried rice with vegetables?)z6How do async and await work with coroutines in Python?z<What is the role of event loops in asynchronous programming?zUCan you explain the differences between async/await and traditional callback methods?z:How do coroutines interact with event loops in JavaScript?zDWhat are the benefits of using async await over promises in Node.js?zECan you nest async functions and how does that affect the event loop?z4What is the performance impact of using async/await?r   r>   r   )rj  r  r#   rW  rZ   randomcopyshuffler   r   rm  _validation_queries)rK   r   r  r  n_totalpromptrh  ri  
variationsr  other_queriesn_validationval_queriestrain_queriestrain_embeddingss                  r&   map_query_semantic_space*EmbeddingStrategy.map_query_semantic_space  s-    : kC'( wi'bchbi j+ . MQOO4??&&z3GHau8<DOO''4T	   !K  L
   "
 
& 	 #9-224}% 1c#m"4s":;<#MN3-,"?? "&!5!5m!DD $/ .. Es   C3D5D6Dquery_pointsalphac                 &    SSK n[        U5      S:  a  gUR                  U5      nUR                  USS9UR	                  USS9UUR                  UR                  R                  XR                  USS9-
  SS95      S.nU$ ! [         a     gf = f)zEFind the minimal shape that covers all query points using alpha shaper   Nr   rq  r   )centerstdpointsradius)	rY   r   r[   meanr  r   ru  rv  	Exception)rK   r  r  r`   r   s        r&   compute_coverage_shape(EmbeddingStrategy.compute_coverage_shape  s    	< 1$
 88L1L '',Q'7vvlv3&&&ww|Z[w?\0\cd!ef	H O 		s   B A,B 
BB	n_samplesc                     SSK nUc  / $ / $ )z*Sample points from the boundary of a shaper   N)rY   )rK   rs  r  r`   s       r&   _sample_boundary_points)EmbeddingStrategy._sample_boundary_points  s     =I 	r%   c                 H   SSK n/ nUb  [        U5      S:X  a  U H  nUR                  US45        M     U$ U R                  X!5      nUc  U H  nUR                  US45        M     U$ UR	                  USS9n[        U5       H  u  pUR                  XWU   45        M     U$ )zLCalculate gap distances for all query variations using vectorized operationsr   Nr   r   r  )rY   r   r   r  r   	enumerate)	rK   r2   r3   r`   gapsq_embr|  min_distancesr  s	            r&   find_coverage_gaps$EmbeddingStrategy.find_coverage_gaps  s     C$6!$;)UCL) *K ::;K[")UCL) *K Q7 ""23HAKKa 012 4 r%   candidate_linksr  c                   ^%#    SSK JnJnJn  SSKnSSKn/ n	/ n
/ n0 nU GHf  nSR                  [        SUR                  =(       d    SUR                  =(       d    S[        US5      (       a-  UR                  (       a  UR                  R                  SS5      OSUR                  (       a,  UR                  R                  S0 5      R                  SS5      OS/5      5      nUR                  5       (       d  M  UR                  UR                    S	U 3R#                  5       5      R%                  5       nXR&                  ;   a  U R&                  U   XR                   '   GMD  U
R)                  U5        UR)                  U5        GMi     U(       a  S
[*        R,                  " S5      S.nU" UUU R.                  5      I Sh  vN n[1        XU5       H`  u  nnnUR                  UR                    S	U 3R#                  5       5      R%                  5       nUU R&                  U'   UXR                   '   Mb     [        U S5      (       a  U R2                  R4                  OSm%U GH  nUR                   U;  a  M  XR                      nU(       d  SnGOLSnSnU H3  u  nnUT%:  d  M  U" UU5      nUU:  d  M  UU-
  nUS-  nUU-  nUS-  nM5     [7        U%4S jU 5       5      nUS:  a  UU-  nOSnUb  [9        U5      S:  a  UUR:                  R=                  U5      -  nX7R:                  R=                  USSS9-  n UR?                  U U5      n!URA                  U!5      n"[        U S5      (       a  U R2                  RB                  OSn#U"U#:  a	  U"U#-
  S-  n$OSn$OSn$USU$-
  -  n[        US5      (       a&  URD                  (       a  US-  URD                  S-  -   nU	R)                  UU45        GM     [G        U	S SS9$  GN?7f)z0Select links that most efficiently fill the gapsr   )cosine_distancecosine_similarityrd  r   Nr'  r   r(  r*  :re  rf  rg  r   r   r   r>   c              3   <   >#    U  H  u  pUT:  d  M  S v   M     g7f)r   Nr$   ).0_r^   coverage_radiuss      r&   	<genexpr>?EmbeddingStrategy.select_links_for_expansion.<locals>.<genexpr>s  s     'Rddaa/>Qds   	Trp  r   r0  r   c                     U S   $ r  r$   r  s    r&   r'   >EmbeddingStrategy.select_links_for_expansion.<locals>.<lambda>  s    !A$r%   r  )$rj  r  r  rd  rY   hashlibr,  r-  r.  r)  rj   r(  rZ   r/  stripmd5r  encode	hexdigestrZ  r   rk  rl  r7   zipr   r   r   r   ru  rv  rw  r   r   r0  sorted)&rK   r  r  r2   r  r  rd  r`   r  r   links_to_embedtexts_to_embedlink_embeddings_maprM   r1  	cache_keyr   new_embeddingsr.  	embeddinglink_embeddingr$  gaps_helpedtotal_improvement	gap_pointgap_distancenew_distanceimprovementscaled_improvementgaps_needing_helpgap_reduction_score	link_normrz  similaritiesmax_similarityoverlap_thresholdoverlap_penaltyr  s&                                        @r&   select_links_for_expansion,EmbeddingStrategy.select_links_for_expansion  s     	SR  #D		R

 b4;D&4I4Idii		mR0]_IM""62.22="E]_	/ " I ??$$  tyyk9+$>$E$E$GHRRTI 666151K1KI1V#II.%%d+%%i0) $. ;YY'78$  $7~G[]a]q]q#rrN *-^^)\%dI#KK499+Qtf(=(D(D(FGQQS	8A**951:#II. *] DK4QYCZCZ$++??`c $Dyy 330;N  $%!/3+I|#o5'6~y'Q',6*6*EK1<q.-1CC-'1,K 04 %('Rd'R$R!$q(*;>O*O'*+' !,]1Ca1G .1O OI+iinn]QR]an.bbG $&66'9#=L%'VVL%9N T[[_aiSjSj(O(Opt%%(99+9<M+MQR*R*+&'O ,q?/BC 4!3449N9N!CK$*?*?#*EEEu.u $x lEEQ ss    F?PPCP%P6EPr]   c                 J  #    SSK nUR                  b  UR                  c  g[        UR                  5      S:X  d  [        UR                  5      S:X  a  gUR	                  UR                  UR
                  S9nUR	                  UR                  UR
                  S9nX2R                  R                  USSS9S-   -  nXBR                  R                  USSS9S-   -  nX4R                  -  R                  SS	9n[        U R                  S
S5      nUb  [        XV:  R                  5       5      O[        UR                  5       5      nXqR                  S'   [        UR                  5       5      UR                  S'   [        UR                  U5      5      UR                  S'   U$ 7f)u&   Coverage-based learning score (0–1).r   Nr   )dtyper   Trp  g:0yE>r  coverage_taucoverage_scoreavg_best_similaritymedian_best_similarity)rY   r2   r3   r   asarrayfloat32ru  rv  rx  r   getattrr   r   r  r   median)rK   r]   r`   QDbesttaur$  s           r&   r   &EmbeddingStrategy.calculate_confidence  sb     &%*@*@*Hu""#q(C0F0F,G1,L JJu--RZZJ@JJu**"**J=	YY^^AA^5<<	YY^^AA^5<< CC}}!}$ dkk>48/2t{((*+E$))+DV +0&'/4TYY[/A+,27		$2H./s   F!F#r   c                   #    X l         [        5       n/ nUR                   H]  nUR                  UR                  ;  d  M  UR                  U;  d  M1  UR                  U5        UR                  UR                  5        M_     U(       d  / $ U R                  UR                  UR                  5      nU Vs/ s H  owS   R                  5       US   4PM     snUl        U R                  UUUR                  5      I Sh  vN $ s  snf  N	7f)z!Main entry point for link rankingr   r   N)r   r-   r   r  r   r   rC  r  r2   r3   rG   r6   r  )rK   r]   r   	seen_urlsuncrawled_linksrM   r  gs           r&   r   EmbeddingStrategy.rank_links  s       E	''Dyy 2 22tyy	7Q&&t,dii( (
 I &&""
 ?CCd!qt4dC 44
 
 	
 D
s*   :D
 D
A#D
5!D(D
>D?
D
c                 ~  #    [        U S5      (       a  U R                  (       d  UR                  R                  SS5      $ SSKnU R
                  c(  U R                  U R                  5      I Sh  vN U l        U R
                  nUR                  b  [        UR                  5      S:X  a  gU R                  X1R                  5      nUc  gUR                  USS9n[        U S5      (       a  U R                  R                  OS	nUR                  U* U-  5      nUR                  U5      nXR                  S
'   U$  N7f)z5Validate coverage using held-out queries with cachingr  r   r   r   Nr   r  r   r   validation_confidence)rj   r  r   rZ   rY   r^  rm  r2   r   r}  r   r   r   expr  )	rK   r]   r`   val_embeddingsr|  r  k_expscoresr  s	            r&   validate_coverage#EmbeddingStrategy.validate_coverage  s    t2334;S;S==$$\377 ,,46:6J6J4KcKc6d0dD-:: &#e.A.A*Ba*G 77H[H[\" Q7 07tX/F/F++C./ "1F-.$$1 1es   A.D=0D;1CD=c                 D  #    UR                   R                  SS5      n[        UR                  5      UR                  :  d  UR
                  (       d  g[        US5      (       d  / Ul        UR                  R                  U5        [        UR                  5      S:  a  g[        [        UR                  SS UR                  S	S 5      5      n[        S
 U 5       5      [        U5      -  nXQR                   S'   [        U S5      (       a  U R                  R                  U-  OSU-  nXV:  am  U R                  U5      I Sh  vN n[        U S5      (       a  U R                  R                  OSnXx:  a  SUR                   S'   SU l        gSUR                   S'   g NY7f)z(Stop based on learning curve convergencer   r   Tconfidence_historyr>   FNr  r   c              3   @   #    U  H  u  p[        X!-
  5      v   M     g 7fr!   )r   )r  abs      r&   r  0EmbeddingStrategy.should_stop.<locals>.<genexpr>T  s     G5FTQc!%jj5Fs   avg_improvementr   r   r   converged_validatedstopped_reasonlow_validation)r   rZ   r   r   r   r   rj   r  r   rB   r  r   r   r   r  r   r[  )	rK   r]   r   r   improvement_diffsr  min_relative_improvement	val_scorevalidation_mins	            r&   r   EmbeddingStrategy.should_stop?  s    ]]&&|S9
 u!!"f&6&66e>Q>Q u233')E$  ''
3 u''(1, U%=%=cr%BED\D\]^]_D`!ab G5FGG#N_J``+:'(bijnpxbyby4;;#Q#QT^#^  @C  FP  @P 5"44U;;I LSSWYaKbKbT[[GGhkN)2G./*.'2B./  <s   EF FAF c                 6   UR                   R                  SS5      nUR                   R                  SS5      n[        U S5      (       a  U R                  R                  OSn[        U S5      (       a  U R                  R
                  OSn[        U S5      (       a  U R                  R                  OSn[        U S5      (       a  U R                  R                  OSnU R                  (       a&  X4:  a!  US:  a  UnU$ US:  a  UnU$ XRS-
  U-  -   n U$ US	-  nU$ )
z4Calculate quality-based confidence score for displaylearning_scorer   r  r   r   r   r   r   r   )	r   rZ   rj   r   r   r   r   r   r[  )	rK   r]   r  validation_scorer  quality_minquality_maxscale_factorr   s	            r&   get_quality_confidence(EmbeddingStrategy.get_quality_confidenceh  s   **+;SA ==,,-DcJ HOtU]G^G^CCdgFMdT\F]F]dkkBBcfFMdT\F]F]dkkBBcgELTS[E\E\t{{AAbg""'7'H #(
   #%(
  )S,@L+PP

  (#-Jr%   r   c                    #    SSK Jn  SSKn/ n/ nU Hk  n[        US5      (       a'  UR                  (       a  UR                  R
                  OSnU(       d  MF  UR                  USS 5        UR                  U5        Mm     U(       d  gS[        R                  " S	5      S
.n	U" XYU R                  5      I Sh  vN n
UR                  c$  Xl
        [        [        [        U
5      5      5      nO/ n/ n[        U
5       H  u  pXR                  R!                  U5      -  nUR                  UR                  R!                  UR                  SSS9-  nUR#                  UU5      nUR%                  U5      U R&                  :  d  M  UR                  U5        UR                  U5        M     U(       a1  UR)                  UR                  UR+                  U5      /5      Ul
        U H+  nUR,                  R                  UU   R.                  5        M-     SU l        SU l        [        US5      (       aW  UR4                  bI  U R7                  UR4                  [        U S5      (       a  U R8                  R:                  OS5      Ul        ggg GN7f)z9Update embeddings and coverage metrics with deduplicationr   rc  r   Nrc   r   i  re  rf  rg  Trp  r3   r   r   )rj  rd  rY   rj   rc   rd   r   rk  rl  r7   r2   rB   r  r   r  ru  rv  rw  r   r_  vstackr[   r1   rh   r]  r\  r3   r  r   r   r5   )rK   r]   r   rd  r`   	new_textsvalid_resultsrD  ri   r   r  deduplicated_indicesdeduplicated_embeddingsr  new_embnew_emb_normalizedkb_normalizedr  idxs                      r&   r   EmbeddingStrategy.update_state  s\    . 	!F6=fj6Q6QV\VeVefoo22kmGw  $0$$V,	 "  8#34 
  39TXThThii &"0#'c..A(B#C  ')##% '7
%,yy~~g/F%F" % 3 3biinnUEXEX_`kon6p p!vvm5GH 66,'$*G*GG+227;(//2 8 '&(ii1D1DbhhOfFg0h&i# (C$$]3%7%;%;< ( $( &*# 5,--%2H2H2T#'#>#>u?U?Ux  AE  GO  yP  yPW[WbWbWtWt  VY  $ZE  3U-I js!   AJA J9J:CJDJ)r\  rY  r]  r_  rZ  r^  r[  r  r   r7   rW  )NN)r   )r   )r   )!ry   rz   r{   r|   r   rk   r   ru   r   r
   rm  r}  r  r#   r  r   r  r  r	   r  r   r  r   r   r   r   r  r   r   r
  r   r   r}   r$   r%   r&   rV  rV  L  s   I- - -
49 
 
 S UX 4+C +PS +X[ +$E/C E/c E/SV E/N3 u 2
 
T#Y 
 s tTYZ]_dZdTeOf <vFdvF 5e$%vF 	vF
 
eD%K 	!vFp
 u L
j 
. 
TRWX\^cXcRdMe 
>!%Z !%E !%F'z '> 'd 'RJ 5 6;Z
 ;ZkAR ;ZW[ ;Zr%   rV  c            	          \ rS rSrSr   S#S\\   S\\   S\\   4S jjr	S\
S	\4S
 jr S$S\
S\
S\\
   S	\4S jjrS\
S\
S	\\   4S jrS\\\\4      S\
S	\\   4S jr\S	\4S j5       r\S	\\
\4   4S j5       r\S	\4S j5       rS%S\S	S4S jjrS	\
4S jrS&S\\
\4   S\
S	S4S jjr S	\\
\4   4S jr!S&S\\
\4   S\
S	S4S jjr"S\\
\4   4S jr#S'S \$S	\\\
\4      4S! jjr%S"r&g)(AdaptiveCrawleri  z<Main adaptive crawler that orchestrates the crawling processNcrawlerr   r   c                    Xl         U=(       d
    [        5       U l        U R                  R                  5         U(       a  X0l        O*U R                  U R                  R                  5      U l        S U l        US L U l        g r!   )r  r   r   r   r   _create_strategyr]   _owns_crawler)rK   r  r   r   s       r&   ru   AdaptiveCrawler.__init__  se     0 0 $M 11$++2F2FGDM ,0
 %_r%   strategy_namerT   c                     US:X  a
  [        5       $ US:X  a2  [        U R                  R                  U R                  R                  S9$ [        SU 35      e)z&Create strategy instance based on namer   r  )r7   rW  zUnknown strategy: )r   rV  r   r7   r   
ValueError)rK   r  s     r&   r   AdaptiveCrawler._create_strategy  sW    M)&((k)$ $ ; ;;;;; 
 1-ABBr%   	start_urlr   resume_fromc           	        #    U(       a+  [         R                  U5      U l        X R                  l        O[        [	        5       / / U0 S9U l        U R
                  (       d1  [        5       U l        U R
                  R                  5       I Sh  vN   U R                  U R                  l        [        U R                  [        5      (       a  U(       d  U R                  R                  UU R                  R                  5      I Sh  vN u  pEX@R                  l        USS U R                  l        U R                  R                   U R                  l         XR                  R"                  ;  Ga  U R%                  X5      I Sh  vN nU(       Ga  ['        US5      (       Ga  UR(                  (       Ga  U R                  R*                  R-                  U5        U R                  R"                  R/                  U5        ['        US5      (       Ga  UR0                  (       Ga  [        UR0                  [2        5      (       a  UR0                  R5                  S/ 5       Vs/ s H  n[7        S0 UD6PM     nnUR0                  R5                  S/ 5       Vs/ s H  n[7        S0 UD6PM     n	nU R                  R8                  R;                  X-   5        OPU R                  R8                  R;                  UR0                  R<                  UR0                  R>                  -   5        U R                  RA                  U R                  U/5      I Sh  vN   Sn
XR                  RB                  :  Ga  U R                  RE                  U R                  5      I Sh  vN nXR                  RF                  S	'   U R                  RI                  U R                  U R                  5      I Sh  vN (       a  GOCU R                  RK                  U R                  U R                  5      I Sh  vN nU(       d  GOUS   S   U R                  RL                  :  a  GOUSU R                  RN                    VVs/ s H.  u  p}URP                  U R                  R"                  ;  d  M+  X}4PM0     nnnU(       d  GOU RS                  X5      I Sh  vN nU(       Ga  U R                  R*                  R;                  U5        [U        X5       GHy  u  nu  nnU(       d  M  U R                  R"                  R/                  URP                  5        ['        US5      (       d  MU  UR0                  (       d  Mh  / n[        UR0                  [2        5      (       ao  UR0                  R5                  S/ 5       Vs/ s H  n[7        S0 UD6PM     nnUR0                  R5                  S/ 5       Vs/ s H  n[7        S0 UD6PM     n	nX-   nO-UR0                  R<                  UR0                  R>                  -   nU HN  nURP                  U R                  R"                  ;  d  M)  U R                  R8                  R-                  U5        MP     GM|     U R                  RA                  U R                  U5      I Sh  vN   U
S-  n
U R                  RV                  (       aJ  U R                  RX                  (       a/  U R                  R[                  U R                  RX                  5        XR                  RB                  :  a  GM  U R                  RE                  U R                  5      I Sh  vN n[        U R                  [        5      (       a=  U R                  R]                  U R                  5      U R                  RF                  S	'   OUU R                  RF                  S	'   [_        U R                  R"                  5      U R                  RF                  S
'   XR                  RF                  S'   U R                  RV                  (       aJ  U R                  RX                  (       a/  U R                  R[                  U R                  RX                  5        U R                  U R`                  (       a8  U R
                  (       a&  U R
                  Rc                  SSS5      I Sh  vN   $ $ $  GNT GN GNbs  snf s  snf  GN GN_ GN GNs  snnf  GN>s  snf s  snf  GNg GN N?! U R`                  (       a9  U R
                  (       a'  U R
                  Rc                  SSS5      I Sh  vN    f f f = f7f)z&Main entry point for adaptive crawling)r   r   r   r   r   Nr   successre   internalexternalr   r   pages_crawleddepth_reachedr$   )2r   rW   r]   r   r-   r  r   
__aenter__r   r   
isinstancerV  r  r   r3   r4   r7   r   _crawl_with_previewrj   r%  r   r   rC  re   rE   rZ   r   r   extendr&  r'  r   r   r   r   r   r   r   r   r  _crawl_batchr  r   r   rR   r
  r   r  	__aexit__)rK   r"  r   r#  r3   r4   rD  rM   internal_linksexternal_linksdepthr   ranked_linksr$  to_crawlr   r  	new_links	link_datanew_linkr  s                        r&   digestAdaptiveCrawler.digest  s     #5DJ$JJ# U! DJ ||*,DL,,))+++#{{ dmm%6777;}}7]7]..8 2. +;JJ'*:12*>DJJ')-)F)FDJJ&l	?

 7 77#77	IIgfi88V^^^JJ--44V<JJ++//	:vw//FLLL%fllD99GM||GWGWXbdfGg-hGgtdlTlGgN-hGM||GWGWXbdfGg-hGgtdlTlGgN-h JJ44;;N<[\ !JJ44;;FLL<Q<QTZT`T`TiTi<ij --44TZZ&JJJ E++///#'==#E#Edjj#QQ
3=

""<0 224::t{{KKK &*]]%=%=djj$++%VV#  ?1%(F(FF >JJb4;;KbKb=c H=ckd"iitzz/F/FF *TM=c H   %)$5$5h$FFJJ--44[A .1-G)	q!6 JJ3377		B&vw77FLLL,.	#-fllD#A#AY_YeYeYiYijtvxYy5zYyId6GY6GYyN5zY_YeYeYiYijtvxYy5zYyId6GY6GYyN5z0>0OI 170E0EH]H]0]I 1:H'/}}DJJ<S<S'S(,

(@(@(G(G(Q 1:! .H* --44TZZMMM
 ;;))dkk.D.DJJOODKK$:$:;{ ++///@ $(==#E#Edjj#QQN $--):;;37==3W3WX\XbXb3c

""<0 4B

""<025djj6M6M2NDJJ/27JJ/ {{%%$++*@*@

 6 67:: !!dllll,,T4>>> '3!u ,2 J .i-h K R L  WH G 6{5z N R* ? !!dllll,,T4>>> '3!sX  B
e)c A8e)c#Ae).d c&Cd  c)5 d c.*B$d c3Ad c6Ad 'c9(>d &c<'Ad 2*c? c?' d dAd >d d ,?d +d  d  d5Ad Ad +d,Bd 6(d dD7d Ae)de)#e)&d )d 6d 9d <d ?d d d e)Ae&e e&&e)rh   c                   #    [        [        SSUSSSSS9SS9n U R                  R                  XS9I Sh  vN n[	        US	5      (       a   UR
                  (       a  UR
                  S
   n[	        US5      (       aU  UR                  (       aD  UR                  S    Vs/ s H  oUR                  S5      (       d  M  UPM     snUR                  S'   U$  Ns  snf ! [         a  n[        SU SU 35         SnAgSnAff = f7f)z%Crawl a URL with link preview enabledTFr   2   )include_internalinclude_externalr   concurrencytimeout	max_linksverbose)link_preview_configscore_links)rh   r   N_resultsr   re   r&  r/  zError crawling z: )
r   r   r  arunrj   rD  re   rZ   r  rB  )rK   rh   r   r   rD  rM   es          r&   r,  #AdaptiveCrawler._crawl_with_previewv  s     ! 1!%!&! 
	<<,,,DDFvz**v+ vw''FLL=C\\*=U+o=UTYaYabmYnD=U+oZ( M E ,p
  	OC51#./	sW   DC CA(C  C=CC DC C 
D&C<7D<DDlinks_with_scoresc                   #    / nU H2  u  pEU R                  UR                  U5      nUR                  U5        M4     [        R                  " USS06I Sh  vN n/ nU H  n	[        U	[        5      (       ab  [        U	S5      (       a$  U	R                  (       a  UR                  U	5        MM  [        S[        U	S5      (       a  U	R                  OS 35        Mz  [        U	[        5      (       d  M  [        SU	 35        M     U$  N7f)	zCrawl multiple URLs in parallelreturn_exceptionsTNr%  zSkipping failed crawl: rh   unknownzError in batch crawl: )r,  r  r   asynciogatherr+  r   rj   r%  rB  rh   r  )
rK   rH  r   tasksrM   r$  taskresultsr  rD  s
             r&   r.  AdaptiveCrawler._crawl_batch  s     ,KD++DIIu=DLL -  FFF F&+..69--&..!((03'&RWBXBXFJJ^g3hijFI...vh78   Gs   ADDBD0Dc                 r    U R                   (       a&  U R                   R                  R                  SS5      $ g)zCurrent confidence levelr   r   )r]   r   rZ   r   s    r&   r   AdaptiveCrawler.confidence  s+     ::::%%)),<<r%   c                 |   U R                   (       d  0 $ [        S U R                   R                   5       5      n[        U R                   R                  5      U[        U R                   R
                  5      [        U R                   R
                  R                  5       5      [        U R                   R                  5      U R                  U R                   R                  R                  SS5      U R                   R                  R                  SS5      U R                   R                  R                  SS5      S.	$ )zDetailed coverage statisticsc              3   r   #    U  H-  n[        UR                  R                  =(       d    S 5      v   M/     g7f)r   N)r   rc   rd   )r  rD  s     r&   r  1AdaptiveCrawler.coverage_stats.<locals>.<genexpr>  s.      #
3 ,,2333s   57r   r   r   r   )	r(  total_content_lengthunique_termstotal_termsr   r   r   r   r   )r]   r   r   r   r   r)   r   r   r   r   rZ   )rK   rW  s     r&   coverage_statsAdaptiveCrawler.coverage_stats  s     zzI" #
**33#
  
 !!8!89$8

 ; ;<tzz::AACD !9!9://

**..z3?::--11-E**,,00sC

 
	
r%   c                     [        U R                  [        5      (       a  U R                  R                  $ U R                  U R
                  R                  :  $ )z(Check if current knowledge is sufficient)r+  r   rV  r[  r   r   r   r   s    r&   is_sufficientAdaptiveCrawler.is_sufficient  sA     dmm%677==333 ??dkk&F&FFFr%   detailedc                   ^  T R                   (       d  [        S5        g SSKJn  SSKJn  U" 5       nSnU(       Gd4  U(       Ga,  W" ST R                   R                   S	3S
9nUR                  SSSS9  UR                  SSS9  T R                  nUR                  S[        UR                  SS5      5      5        UR                  S[        UR                  SS5      5      5        UR                  S[        UR                  SS5      5      5        UR                  SUR                  SS5      S S35        UR                  S[        UR                  SS5      5      5        UR                  SS5        [        T R                  [        5      (       Ga  UR                  SUR                  SS5      S  5        UR                  S!T R                   R                   R                  S"S5      S# 5        UR                  S$T R                   R                   R                  S%S5      S& 5        UR                  S'T R                   R                   R                  S(S5      S  5        UR                  SS5        UR                  S)T R"                  (       a  S*OS+5        OUR                  SUR                  SS5      S  5        UR                  S,UR                  S-S5      S  5        UR                  S.UR                  S/S5      S  5        UR                  S0UR                  S1S5      S  5        UR                  SS5        UR                  S)T R"                  (       a  S2OS+5        WR                  U5        g[        S35        [        S4T R                   R                   S	35        [        S55        [        S65        [        S7[%        T R                   R&                  5       35        [        S8[%        T R                   R(                  5       35        [        S9T R                   R*                   35        [-        U 4S: jT R                   R.                   5       5      n[-        T R                   R0                  R3                  5       5      n	[%        T R                   R0                  5      n
[        S;5        [        S<US S=35        [        S>U	S 35        [        S?U
S 35        U	S:  a  [        S@X-  S  35        [        T R                  [        5      (       Ga  [        SA5        [        SBT R                   R                   R                  S"S5      S# 35        [        SCT R                   R                   R                  S%S5      S& 35        [        SDT R                   R                   R                  SES5      S& 35        [        SF5        T R"                  (       aC  U(       a!  WR                  SGT R4                  S  SH35        O][        SGT R4                  S  SI35        OBU(       a!  WR                  SGT R4                  S  SJ35        O[        SGT R4                  S  SK35        [        SLT R                   R                   R                  SMS5      S  35        [        SNT R                   R                   R                  S(S5      S  35        GO[        SO5        T R                  R7                  T R                   R                  R9                  5       5      nU GH#  nT R                   R0                  R                  US5      nT R                   R:                  R                  US5      nUS:  a  U(       aO  WR                  SPU SQU SRT R                   R*                   SSUT R                   R*                  -  ST SUU SV35        M  [        SPU SQU SRT R                   R*                   SWUT R                   R*                  -  ST SXU SV35        M  U(       a  WR                  SPU SY35        GM  [        SPU SZ35        GM&     [        SF5        T R"                  (       a  S[OS\nU(       a8  T R"                  (       a  S]OS^nWR                  SGT R4                  S  S_U 35        O[        SGT R4                  S  S_U 35        [        S`T R                   R                   R                  S-S5      S  35        [        SaT R                   R                   R                  S/S5      S  35        [        SbT R                   R                   R                  S1S5      S  35        T R                   R<                  (       a  [-        T R                   R<                  5      [%        T R                   R<                  5      -  n[        Sc5        [        SdUS& 35        [        SeT R                   R                   R                  S1S5      S  35        U(       Ga  [        Sf5        U(       a  WR                  Sg5        O[        Sh5        [        Si5        [        Sj5        [?        T R                   R0                  RA                  5       Sk SSl9SSm n[C        USn5       Hp  u  nu  nnT R                   R:                  R                  US5      nU(       a!  WR                  SoUSp SqU SrU SsU St3	5        MW  [        SoUSp SuU SvU SsU St3	5        Mr     [        Sw[%        T R                   R&                  5       Sx35        [C        T R                   RD                  Sn5       H  u  nnU[%        T R                   R<                  5      ::  a  T R                   R<                  USn-
     OSnU(       a/  WR                  SoU SyU Sz35        UR                  S{U S|35        M}  [        SoU S}U 35        [        S~U S35        M     [        S5        0 nT R                   R:                  R3                  5        H  nUR                  US5      Sn-   UU'   M     [?        URG                  5       5       H  nUU   n[        SU SU S35        M     T R                   RH                  (       Gap  [        S5        [        ST R                   RH                   35        [        S[%        T R                   RJ                  5       35        T R                   RL                  b-  [        ST R                   RL                  RN                   35        O[        S5        [        S[%        T R                   RP                  5       35        [        ST R4                  S  35        T R                   RJ                  (       aj  [        S5        [C        T R                   RJ                  SS Sn5       H8  u  nnU(       a  WR                  SoU SU S35        M'  [        SoU S}U 35        M:     [        S35        g! [         a    Sn GNf = f)zPrint comprehensive statistics about the knowledge base

Args:
    detailed: If True, show detailed statistics including top terms
zNo crawling state available.Nr   )Console)TableTFzAdaptive Crawl Stats - Query: '')r)  Metriccyan)styleno_wrapValuemagenta)rf  zPages Crawledr(  zUnique TermsrX  zTotal TermsrY  zContent LengthrW  ,z charszPending Linksr   r   
Confidencer   z.2%zAvg Min Distanceavg_min_distancez.3fzAvg Close Neighborsavg_close_neighborsz.1fzValidation Scorer  zIs Sufficient?z[green]Yes (Validated)[/green]z[red]No[/red]Coverager   Consistencyr   
Saturationr   z[green]Yes[/green]zQ
================================================================================z$Adaptive Crawl Statistics - Query: 'zP================================================================================z
[*] Basic Statistics:z  Pages Crawled: z  Pending Links: z  Total Documents: c              3   X   >#    U  H  n[        TR                  U5      5      v   M!     g 7fr!   )r   _get_content_from_result)r  rD  rK   s     r&   r  .AdaptiveCrawler.print_stats.<locals>.<genexpr>  s+      '7F D11&9::7s   '*z
[*] Content Statistics:z  Total Content: z charactersz  Total Words: z  Unique Terms: z  Vocabulary Richness: z 
[*] Semantic Coverage Analysis:z  Average Min Distance: z  Avg Close Neighbors (< 0.3): z$  Avg Very Close Neighbors (< 0.2): avg_very_close_neighborsz
[*] Confidence Metrics:z  Overall Confidence: z [green][VALIDATED][/green]z [VALIDATED]z [red][NOT VALIDATED][/red]z [NOT VALIDATED]z  Learning Score: r  z  Validation Score: z
[*] Query Coverage:z  'z': found in /z docs ([green]z.0%z[/green]), z occurrencesz docs (z), z': [red][X] not found[/red]z': [X] not foundz[OK]z[!!]z[green][OK][/green]z[red][!!][/red]r'  z  Coverage Score: z  Consistency Score: z  Saturation Score: z
[*] Crawl Efficiency:z  Avg New Terms per Page: z  Information Saturation: zQ
--------------------------------------------------------------------------------z*[bold cyan]DETAILED STATISTICS[/bold cyan]zDETAILED STATISTICSzP--------------------------------------------------------------------------------z
[+] Top 20 Terms by Frequency:c                     U S   $ r  r$   r  s    r&   r'   -AdaptiveCrawler.print_stats.<locals>.<lambda>m  s    VWXYVZr%   r  r   r   z  2dz. [yellow]'z'[/yellow]: z occurrences in z docsz. 'z': z
[+] URLs Crawled (z):z. [cyan]z[/cyan]z     -> Added [green]z[/green] new termsz. z     -> Added z
 new termsz%
[+] Document Frequency Distribution:z  Terms in z docs: z termsz 
[+] Semantic Coverage Analysis:z  Embedding Model: z  Query Variations: z  Knowledge Embeddings: z  Knowledge Embeddings: Nonez  Semantic Gaps: z  Coverage Achievement: z
[+] Query Space (samples):r   z
. [yellow]z	[/yellow]))r]   rB  rich.consolera  
rich.tablerb  ImportErrorr   
add_columnrZ  add_rowrk   rZ   r+  r   rV  r   r]  r   r   r   r/   r   r   r)   r   r   r   r   r+   r0   r  rF   r  r1   r5  r7   r4   r2   rs  r6   )rK   r_  ra  rb  consoleuse_richtablestatsrW  total_wordsrX  r   r   r   r   statusstatus_coloredavg_new_terms	top_termsr  freqrh   r7  	df_countscounteqs   `                         r&   print_statsAdaptiveCrawler.print_stats  s9    zz01	,(iGH H"A$**BRBRASST UVEXVTBWI6 ''EMM/3uyy!/L+MNMM.#eii.J*KLMM-UYY}a-H)IJMM*uyy9OQR/STU.VV\,]^MM/3uyy!/L+MNMM"b! $--):;;luyyq/I#.NP0TZZ5G5G5K5KL^`a5bcf4gi3

8J8J8N8NOdfg8hil7mo0TZZ5G5G5K5KLcef5ghk4lnb"%.TXTfTf0Pl{| luyyq/I#.NPjUYYz1-Ec,JLm		-0KC/PRluyyq/I#.NPb"%.HZHZ0D`opMM%  - 89I9I8J!LM&M +,%c$***A*A&B%CDE%c$***B*B&C%DEF'

(B(B'CDE $' '"jj77' $  djj99@@BCKtzz::;L-/%&:1%=[IJOK?34$\!$456Q/0H/MNO $--):;;9;01C1C1G1GHZ\]1^_b0cde7

8J8J8N8NOdfg8hil7mno<TZZ=O=O=S=STnpq=rsv<wxy 13%%(>ts>SSn&op 6ts6K<XY(>ts>SSn&op 6ts6KK[\]*4::+=+=+A+ABRTU+VWZ*[\],TZZ-?-?-C-CD[]^-_`c,def -/"mm55djj6F6F6L6L6NO'D4488qAB88<<T1EBAv##MMCv\"QtzzGaGaFbbpqstxt~t~  uO  uO  rO  PS  qT  T_  `b  _c  co  +p  q!Cv\"Qtzz?Y?Y>ZZabdeieoeoeeb  AD  bE  EH  IK  HL  LX  #Y  Z##MMCv5P*QR!Cv-=">? ( 13#'#5#56>B>P>P%:VgNMM$:4??3:OqQ_P`"ab24??32GqQR*4::+=+=+A+A*a+PQT*UVW-djj.@.@.D.D]TU.VWZ-[\],TZZ-?-?-C-CLRS-TUX,YZ[ zz++ #DJJ$@$@ AC

HdHdDe e/12=2EFG24::3E3E3I3I,XY3Z[^2_`am$MM"NO/0f 89"4::#>#>#D#D#FNdhijmkmn	'0A'>OA|d88<<T1EB1R&D6dVScdfcggl&mn1R&D6TF:J2$eTU (? ,S1H1H-I,J"MN'

(>(>BFAsEF#djjNjNjJkEk

 < <QqS AqrI1#XcU'&BC(=i[HZ&[\1#Ruo.ykDE C >?	**99@@BB$-MM"a$81$<IbM C !!12B%bMEKt75'@A 3
 ::---=>/

0J0J/KLM0TZZ5P5P1Q0RSTzz//; 89Q9Q9W9W8XYZ <>-c$**2J2J.K-LMN4T__S4IJK zz22<=%.tzz/J/J2A/NPQ%REAr' '1#Zt9.M N %1#Rtn 5	 &S - s  	H	s   w6 6xxc                     [        US5      (       a`  UR                  (       aO  [        UR                  S5      (       a  UR                  R                  =(       d    S$ [        UR                  5      $ g)z,Helper to safely extract content from resultrc   rd   r   )rj   rc   rd   rk   )rK   rD  s     r&   rr  (AdaptiveCrawler._get_content_from_result  sM    6:&&6??v77339r9v''r%   filepathformatc           	      6   U R                   (       a  U R                   R                  (       d  [        S5        g[        U5      nUR                  R                  SSS9  US:X  a  [        USSS9 nU R                   R                   H;  nU R                  U5      nUR                  [        R                  " US	S
9S-   5        M=     SSS5        [        S[        U R                   R                  5       SU 35        g[        SU 35      e! , (       d  f       NK= f)zExport the knowledge base to a file

Args:
    filepath: Path to save the file
    format: Export format - currently supports 'jsonl'
zNo knowledge base to export.NTr:   jsonlr=   utf-8encodingF)ensure_ascii
z	Exported z documents to zUnsupported export format: )r]   r   rB  r   r@   rA   rH   _crawl_result_to_export_dictwriterI   dumpsr   r   )rK   r  r  rQ   rD  result_dicts         r&   export_knowledge_base%AdaptiveCrawler.export_knowledge_base  s     zz!:!:01>dT:Whg6!"jj77F"&"C"CF"KKGGDJJ{G$NO	 8 7 Ic$**";";<=^H:VW:6(CDD 76s   /AD


Dc                 B   [        USS5      [        USS5      [        USS5      U R                  (       a  U R                  R                  OSS.n[        US5      (       a_  UR                  (       aN  [        UR                  S	5      (       a  UR                  R
                  US
'   O[        UR                  5      US
'   OSUS
'   [        US5      (       a  UR                  US'   [        US5      (       a  UR                  US'   U R                  (       a  US   U R                  R                  ;   a+  U R                  R                  R                  US   5      S-   OSU R                  R                  R                  SS5      U R                  R                  S.US'   U$ )z.Convert CrawlResult to a dictionary for exportrh   r   	timestampNr%  T)rh   r  r%  r   rc   rd   ri   rf   re   r   r   r   )r1   confidence_at_crawlr/   crawl_metadata)r  r]   r   rj   rc   rd   rk   rf   re   r1   indexr   rZ   r/   )rK   rD  export_dicts      r&   r  ,AdaptiveCrawler._crawl_result_to_export_dict  sj    65"- d;vy$7)-TZZ%%	
 6:&&6??v77)/)E)EI&),V__)=I&%'K	" 6:&&&,ooK
# 67###)<<K  ::WbchWimqmwmw  nD  nD  XDtzz55;;K<NORSS  JK'+zz'9'9'='=lA'N#'::#=#=-K() r%   c                    [        U5      nUR                  5       (       d  [        SU 35      eUS:X  Ga  / n[        USSS9 nU HR  nUR	                  5       (       d  M  [
        R                  " U5      nU R                  U5      nUR                  U5        MT     SSS5        U R                  (       d  [        5       U l	        U R                  R                  R                  U5        [        R                  " U R                  R!                  U R                  U5      5        [#        S[%        U5       SU 35        g['        S	U 35      e! , (       d  f       N= f)
zImport a knowledge base from a file

Args:
    filepath: Path to the file to import
    format: Import format - currently supports 'jsonl'
zFile not found: r  rV   r  r  Nz	Imported z documents from zUnsupported import format: )r   existsFileNotFoundErrorrH   r  rI   loads_import_dict_to_crawl_resultr   r]   r   r   r-  rL  runr   r   rB  r   r   )rK   r  r  imported_resultsrQ   linedatamock_results           r&   import_knowledge_base%AdaptiveCrawler.import_knowledge_base  s    >  #&6xj$ABBW!hg6!Dzz||#zz$/&*&G&G&M(//<  7 ::'\
 JJ%%,,-=> KK224::?OPQIc"2344DXJOP:6(CDD) 76s   E <E
Er  c                 B   ^  " S S5      m " U4S jS5      nU" U5      $ )z0Convert imported dict back to a mock CrawlResultc                       \ rS rSrS rSrg)BAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockMarkdowni  c                     Xl         g r!   rs   rt   s     r&   ru   KAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockMarkdown.__init__  rw   r%   rs   Nrx   r$   r%   r&   r~   r    r   r%   r~   c                   "   > \ rS rSrU 4S jrSrg)EAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockCrawlResulti  c                 $  > UR                  SS5      U l        T" UR                  SS5      5      U l        UR                  S0 5      U l        UR                  S0 5      U l        UR                  SS5      U l        UR                  S5      U l        g )	Nrh   r   ri   re   rf   r%  Tr  )rZ   rh   rc   re   rf   r%  r  )rK   r  r~   s     r&   ru   NAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockCrawlResult.__init__  sr    88E2. ,TXXi-D E!XXgr2
 $R 8#xx	48!%+!6r%   )re   rc   rf   r%  r  rh   Nrx   r   s   r&   r   r    s    7 7r%   r   r$   )rK   r  r   r~   s      @r&   r  ,AdaptiveCrawler._import_dict_to_crawl_result  s$    	, 	,	7 	7 t$$r%   top_kc                    U R                   (       a  U R                   R                  (       d  / $ / n[        U R                   R                  R	                  5       R                  5       5      n[        U R                   R                  5       H  u  pEUR                  R                  =(       d    SR	                  5       n[        UR                  5       5      n[        X7-  5      nU(       a  U[        U5      -  OSn	UR                  UR                  U	UR                  R                  US.5        M     UR                  S SS9  USU $ )z'Get most relevant content for the queryr   r   )rh   r$  ri   r  c                     U S   $ )Nr$  r$   r  s    r&   r'   6AdaptiveCrawler.get_relevant_content.<locals>.<lambda>6  s    qzr%   Tr  N)r]   r   r-   r   r   rN  r  rc   rd   r   r   rh   r  )
rK   r  scored_docsr   r  rD  ri   content_termsr  r$  s
             r&   get_relevant_content$AdaptiveCrawler.get_relevant_content  s   zz!:!:I $****00288:;"4::#<#<=IA339r@@BG0M +56G2=Gc+..3Ezz!??77	   >  	14@6E""r%   )r  r   r  r]   r   )NNNr!   )F)r  )r   )'ry   rz   r{   r|   r   r   r   r   r   ru   rk   r  r   r8  r   r,  r   r	   r   r   r.  propertyr   r   r
   rZ  r   r]  r  rr  r   r   r  r  r  r  r#   r  r}   r$   r%   r&   r  r    s   F 7;4859-"?3-!.1- $M2-(
Cc 
Cm 
C =AP?*-P?&)P? -5SMP? FPP?dS  +AV @DtU{9K4L UX ]abm]n 0 E   
S#X 
 
, Gt G GI!D I!T I!V# EeCI.> E EZ^ E6#d38n #J!EeCI.> !E !EZ^ !EF%c3h %"## #d4S>6J # #r%   r  )(r   abcr   r   typingr   r   r   r   r	   r
   r   dataclassesr   r   rL  picklerk  rI   r   collectionsr   r   rL  pathlibr   crawl4ai.async_webcrawlerr   crawl4ai.async_configsr   r   crawl4ai.modelsr   r   r   r   r   r   rV  r  r$   r%   r&   <module>r     s    $ ? ? ? (   	   , 	  5 F - {
 {
 {
| ]j ]j ]j@C 0w/- w/t	r	Z r	Zjv	# v	#r%   