
    MiT                    b   S r SSKJrJr  SSKJrJrJrJrJ	r	J
r
Jr  SSKJrJr  SSKrSSKrSSKrSSKrSSKrSSKJrJr  SSKrSSKJr  SSKJr  SS	KJrJrJr  SS
K J!r!J"r"  SSK#r$\ " S S5      5       r%\ " S S5      5       r& " S S\5      r' " S S\'5      r( " S S\'5      r) " S S5      r*g)a  
Adaptive Web Crawler for Crawl4AI

This module implements adaptive information foraging for efficient web crawling.
It determines when sufficient information has been gathered to answer a query,
avoiding unnecessary crawls while ensuring comprehensive coverage.
    )ABCabstractmethod)DictListOptionalSetTupleAnyUnion)	dataclassfieldN)defaultdictCounter)Path)AsyncWebCrawler)CrawlerRunConfigLinkPreviewConfig	LLMConfig)LinkCrawlResultc                   R   \ rS rSr% Sr\" \S9r\\	   \
S'   \" \S9r\\   \
S'   \" \S9r\\   \
S'   Sr\	\
S'   \" \S9r\\	\4   \
S	'   \" S
 S9r\\	\4   \
S'   \" S S9r\\	\4   \
S'   \" S S9r\\	\\   4   \
S'   Sr\\
S'   \" \S9r\\   \
S'   \" \S9r\\	   \
S'   Sr\\   \
S'   Sr \\   \
S'   \" \S9r!\\	   \
S'   Sr"\\   \
S'   \" \S9r#\\$\\   \4      \
S'   Sr%\	\
S'   S\&\	\'4   4S jr(\)S\&\	\'4   SS 4S j5       r*\+S\S\4S  j5       r,\+S!\4S" j5       r-S#r.g)$
CrawlState   z-Tracks the current state of adaptive crawling)default_factorycrawled_urlsknowledge_basepending_links querymetricsc                       [        [        5      $ Nr   int     s/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/adaptive_crawler.py<lambda>CrawlState.<lambda>$   s
    [QTEUr&   term_frequenciesc                       [        [        5      $ r"   r#   r%   r&   r'   r(   r)   %   s
    UXIYr&   document_frequenciesc                       [        [        5      $ r"   )r   setr%   r&   r'   r(   r)   &   s
    kZ]N^r&   documents_with_termsr   total_documentsnew_terms_historycrawl_orderNkb_embeddingsquery_embeddingsexpanded_queriescoverage_shapesemantic_gapsembedding_modelpathc           
         [        U5      nUR                  R                  SSS9  0 S[        U R                  5      _SU R
                   Vs/ s H  o R                  U5      PM     sn_SU R                   Vs/ s H  o3R                  5       PM     sn_SU R                  _SU R                  _S[        U R                  5      _S	[        U R                  5      _S
U R                  R                  5        VVs0 s H  u  pEU[        U5      _M     snn_SU R                   _SU R"                  _SU R$                  _SU R&                  b  U R&                  R)                  5       OS_SU R*                  b  U R*                  R)                  5       OS_SU R,                  _SU R.                  _SU R0                  _n[3        US5       n[4        R6                  " XgSS9  SSS5        gs  snf s  snf s  snnf ! , (       d  f       g= f)z"Save state to disk for persistenceTparentsexist_okr   r   r   r   r    r*   r,   r/   r0   r1   r2   r3   Nr4   r5   r7   r8   w   )indent)r   parentmkdirlistr   r   _crawl_result_to_dictr   
model_dumpr   r    dictr*   r,   r/   itemsr0   r1   r2   r3   tolistr4   r5   r7   r8   openjsondump)selfr9   crlinkkv
state_dictfs           r'   saveCrawlState.save5   s   Dz$6
D!2!23
H[H[\H["99"=H[\
 D<N<NO<NDoo/<NO
 TZZ	

 t||
 T%:%: ;
 #D)B)B$C
 #D<U<U<[<[<]$^<]DAQQZ<]$^
 t33
  !7!7
 4++
 D<N<N<ZT//668`d
 $BWBWBc 5 5 < < >im
  5 5
  T//!
" t33#

( $_IIjA. _% ]O
 %_ _s   G"8G'8G,
G22
H returnc                 $   [        U5      n[        US5       n[        R                  " U5      nSSS5        U " 5       n[	        WS   5      Ul        US    Vs/ s H  oPR                  U5      PM     snUl        US    Vs/ s H  n[        S0 UD6PM     snUl	        US   Ul
        US   Ul        [        [        US   5      Ul        [        [        US	   5      Ul        [        [        US
   R!                  5        VVs0 s H  u  pxU[	        U5      _M     snn5      Ul        US   Ul        US   Ul        US   Ul        UR+                  S5      b  [,        R.                  " US   5      OSUl        UR+                  S5      b  [,        R.                  " US   5      OSUl        UR+                  S/ 5      Ul        UR+                  S/ 5      Ul        UR+                  SS5      Ul        U$ ! , (       d  f       GN= fs  snf s  snf s  snnf )zLoad state from diskrNr   r   r   r   r    r*   r,   r/   r0   r1   r2   r3   r4   r5   r7   r8   r   r%   )r   rI   rJ   loadr.   r   _dict_to_crawl_resultr   r   r   r   r    r   r$   r*   r,   rG   r/   r0   r1   r2   getnparrayr3   r4   r5   r7   r8   )	clsr9   rR   rQ   stated	link_dictrO   rP   s	            r'   rX   CrawlState.loadR   s    Dz$_1J   N!;<FPQaFbcFb 9 9! <FbcBL_B]^B]Yt0i0B]^ )"9-!,S*=O2P!Q%0jAW6X%Y"%0ZXnMoMuMuMw6xMwTQq#a&yMw6x%y" *+< =",-@"A&}5 HR~~VeGfGrbhhz/'BCx|MW^^\nMoM{*5G*H!I  BF!+0BB!G(nn_bA */@" E1 _
  d^
 7ys   G0HHH0
G?rM   c                 r   Sn[        U S5      (       aX  U R                  (       aG  [        U R                  S5      (       a  U R                  R                  nO[        U R                  5      nU R                  U[        U S5      (       a  U R
                  O0 [        U S5      (       a  U R                  S.$ 0 S.$ )z(Convert CrawlResult to serializable dictr   markdownraw_markdownlinksmetadataurlcontentre   rf   )hasattrrc   rd   strrh   re   rf   )rM   markdown_contents     r'   rD    CrawlState._crawl_result_to_dictp   s     2z""r{{r{{N33#%;;#;#; #&r{{#3  66'!(W!5!5RXX2'.r:'>'>	
 	
 EG	
 	
r&   r_   c           	         ^  " S S5      m " U4S jS5      nU" U S   U R                  SS5      U R                  S0 5      U R                  S	0 5      S
9$ )z Convert dict back to CrawlResultc                       \ rS rSrS rSrg)6CrawlState._dict_to_crawl_result.<locals>.MockMarkdown   c                     Xl         g r"   rd   rL   ri   s     r'   __init__?CrawlState._dict_to_crawl_result.<locals>.MockMarkdown.__init__       $+!r&   rs   N__name__
__module____qualname____firstlineno__ru   __static_attributes__r%   r&   r'   MockMarkdownrp          ,r&   r~   c                   "   > \ rS rSrU 4S jrSrg)9CrawlState._dict_to_crawl_result.<locals>.MockCrawlResult   c                 D   > Xl         T" U5      U l        X0l        X@l        g r"   )rh   rc   re   rf   )rL   rh   ri   re   rf   r~   s        r'   ru   BCrawlState._dict_to_crawl_result.<locals>.MockCrawlResult.__init__   s     ,W 5"
 (r&   )re   rc   rf   rh   Nrx   r~   s   r'   MockCrawlResultr      s    ) )r&   r   rh   ri   r   re   rf   rg   )rZ   )r_   r   r~   s     @r'   rY    CrawlState._dict_to_crawl_result   sW    	, 	,	) 	) %EE)R(%%$UU:r*	
 	
r&   r%   )/ry   rz   r{   r|   __doc__r   r.   r   r   rk   __annotations__rC   r   r   r   r   r   r   rF   r    r   floatr*   r$   r,   r/   r0   r1   r2   r3   r   r
   r4   r5   r6   r7   r	   r8   r   r   rS   classmethodrX   staticmethodrD   rY   r}   r%   r&   r'   r   r      s   7"37L#c(7(-d(CND%C %d ;M4:;E3O %d ;GT#u*; (-=U'Vd38nV+0AY+Z$sCx.Z05F^0_$sCH}-_OS $)#>tCy>"48Kc8 $(M8C='&*hsm*"'"=d3i=$(NHSM(5:45PM4d5k5012POS/sDy) /: c4i( \  : 
+ 
$ 
 
" 
 
 
r&   r   c                   ^   \ rS rSr% SrSr\\S'   Sr\	\S'   Sr
\	\S'   S	r\	\S
'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\   \S'   S r\\S!'   Sr\\\\4      \S"'   Sr \\\\4      \S#'   S$r!\	\S%'   S&r"\\S''   Sr#\\S('   Sr$\\S)'   Sr%\\S*'   S+r&\\S,'   Sr'\\S-'   Sr(\\S.'   S&r)\\S/'   S0r*\\S1'   Sr+\\S2'   Sr,\\S3'   Sr-\\S4'   S5r.\\S6'   S7r/\\S8'   S9 r0\1S:\\   4S; j5       r2\1S:\\   4S< j5       r3S=r4g)>AdaptiveConfig   z#Configuration for adaptive crawlingffffff?confidence_threshold   	max_depth   	max_pages   top_k_links皙?min_gain_thresholdstatisticalstrategy皙?saturation_thresholdconsistency_threshold皙?coverage_weight333333?consistency_weightsaturation_weight      ?relevance_weightnovelty_weight皙?authority_weightF
save_stateN
state_path&sentence-transformers/all-MiniLM-L6-v2r8   embedding_llm_configquery_llm_config
   n_query_variations333333?coverage_thresholdalpha_shape_alpha"embedding_min_confidence_thresholdembedding_coverage_radius      ?embedding_k_expembedding_nearest_weightembedding_top_k_weightembedding_overlap_thresholdg      @link_preview_timeout"embedding_min_relative_improvementembedding_validation_min_score embedding_quality_min_confidenceffffff? embedding_quality_max_confidence-?embedding_quality_scale_factorc                    SU R                   s=::  a  S::  d   S5       e   S5       eU R                  S:  d   S5       eU R                  S:  d   S5       eU R                  S:  d   S5       eSU R                  s=::  a  S::  d   S5       e   S5       eU R
                  U R                  -   U R                  -   n[        US-
  5      S	:  d
   S
U 35       eU R                  U R                  -   U R                  -   n[        US-
  5      S	:  d
   SU 35       eSU R                  s=:  a  S:  d   S5       e   S5       eU R                  S:  d   S5       eSU R                  s=::  a  S::  d   S5       e   S5       eSU R                  s=::  a  S::  d   S5       e   S5       e[        U R                  U R                  -   S-
  5      S	:  d   S5       eSU R                   s=::  a  S::  d   S5       e   S5       eSU R"                  s=:  a  S:  d   S5       e   S5       eSU R$                  s=::  a  S::  d   S5       e   S5       eSU R&                  s=::  a  S::  d   S5       e   S5       eSU R(                  s=::  a  S::  d   S5       e   S5       eU R*                  S:  d   S5       eSU R,                  s=::  a  S::  d   S5       e   S5       eg)z!Validate configuration parametersr      z,confidence_threshold must be between 0 and 1zmax_depth must be positivezmax_pages must be positiveztop_k_links must be positivez*min_gain_threshold must be between 0 and 1r   gMbP?z$Coverage weights must sum to 1, got z(Link scoring weights must sum to 1, got z1embedding_coverage_radius must be between 0 and 1z embedding_k_exp must be positivez0embedding_nearest_weight must be between 0 and 1z.embedding_top_k_weight must be between 0 and 1zEmbedding weights must sum to 1z3embedding_overlap_threshold must be between 0 and 1z:embedding_min_relative_improvement must be between 0 and 1z6embedding_validation_min_score must be between 0 and 1z8embedding_quality_min_confidence must be between 0 and 1z8embedding_quality_max_confidence must be between 0 and 1z/embedding_quality_scale_factor must be positivez:embedding_min_confidence_threshold must be between 0 and 1N)r   r   r   r   r   r   r   r   absr   r   r   r   r   r   r   r   r   r   r   r   r   r   )rL   
weight_sums     r'   validateAdaptiveConfig.validate   s(   D--22b4bb2b4bb2~~!?#??!~~!?#??!!#C%CC#D++0q0^2^^0^2^^0 ))D,C,CCdF\F\\
:#$u,a0TU_T`.aa,**T-@-@@4CXCXX
:#$u,e0XYcXd.ee, 4115A5j7jj5j7jj5##a'K)KK'D116Q6j8jj6j8jj6D//414f6ff4f6ff440043N3NNQTTUX]]  	A  `A  	A]D4499p;pp9p;pp94::>Q>|@||>|@||>D77<1<v>vv<v>vv<D99>Q>z@zz>z@zz>D99>Q>z@zz>z@zz>22Q6i8ii6D;;@q@~B~~@~B~~@r&   rU   c                     U R                   c  g[        U R                   [        5      (       a  U R                   $ U R                   R                  5       $ )z<Convert LLMConfig to dict format for backward compatibility.N)r   
isinstancerF   to_dictrL   s    r'   _embedding_llm_config_dict)AdaptiveConfig._embedding_llm_config_dict  sG     $$,d//66,,,((0022r&   c                     U R                   c  g[        U R                   [        5      (       a  U R                   $ U R                   R                  5       $ )z(Convert query LLM config to dict format.N)r   r   rF   r   r   s    r'   _query_llm_config_dict%AdaptiveConfig._query_llm_config_dict  sG       (d++T22((($$,,..r&   r%   )5ry   rz   r{   r|   r   r   r   r   r   r$   r   r   r   r   rk   r   r   r   r   r   r   r   r   r   boolr   r   r8   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   propertyr   r   r}   r%   r&   r'   r   r      s   -"%%%IsIsK ##!Hc! #&%%#&5& OU  ##"u" "e!NE!e! J $J$ DOSC=A(5D#9:A9=huY_56=   $$"u" 14&3 (+u* !OU  '*e)$'E'
 *.-
 #&%% 14&3 -0"E/
 /2$e1.2$e2,1"E1: 3HTN 3 3 / / /r&   r   c            
           \ rS rSrSr\S\S\4S j5       r\S\S\	S\
\\\4      4S j5       r\S\S\	S\4S j5       r\S\S	\
\   SS
4S j5       rSrg
)CrawlStrategyi  z+Abstract base class for crawling strategiesr^   rU   c                    #    g7f)z@Calculate overall confidence that we have sufficient informationNr%   )rL   r^   s     r'   calculate_confidence"CrawlStrategy.calculate_confidence  
      	   configc                    #    g7f)z/Rank pending links by expected information gainNr%   rL   r^   r   s      r'   
rank_linksCrawlStrategy.rank_links  r   r   c                    #    g7f)!Determine if crawling should stopNr%   r   s      r'   should_stopCrawlStrategy.should_stop"  r   r   new_resultsNc                    #    g7f)#Update state with new crawl resultsNr%   )rL   r^   r   s      r'   update_stateCrawlStrategy.update_state'  r   r   r%   )ry   rz   r{   r|   r   r   r   r   r   r   r   r	   r   r   r   r   r   r   r}   r%   r&   r'   r   r     s    5
 u   j . TRWX\^cXcRdMe   z > d   
 kAR W[  r&   r   c            	       ,   \ rS rSrSrS rS\S\4S jrS\S\4S jr	S\S\4S jr
S\S\4S	 jrS\S
\S\\\\4      4S jrS\S\S\4S jrS\S\S\4S jrS\S\4S jrS\S
\S\4S jrS\S\\   SS4S jrS\S\\   4S jrS\S\\   4S jrSrg)StatisticalStrategyi-  z1Pure statistical approach - no LLM, no embeddingsc                 .    0 U l         SU l        SU l        g )Ng333333?g      ?)	idf_cachebm25_k1bm25_br   s    r'   ru   StatisticalStrategy.__init__0  s    r&   r^   rU   c                   #    UR                   (       d  gU R                  U5      nU R                  U5      nU R                  U5      nX!R                  S'   X1R                  S'   XAR                  S'   SU-  SU-  -   SU-  -   nU$ 7f)z@Calculate confidence using coverage, consistency, and saturation        coverageconsistency
saturationr   r   )r   _calculate_coverage_calculate_consistency_calculate_saturationr    )rL   r^   r   r   r   
confidences         r'   r   (StatisticalStrategy.calculate_confidence5  s     ##++E211%8//6
 %-j!'2m$&0l# 8^cK&77#
:JJ
s   BBc                    UR                   (       a  UR                  S:X  a  gU R                  UR                   R                  5       5      nU(       d  g/ nUR                  (       a#  [        UR                  R                  5       5      OSnU H  nUR                  R                  US5      nUR                  R                  US5      nUS:  ag  XqR                  -  nUS:  a3  [        R                  " SU-   5      [        R                  " SU-   5      -  OSn	USSU	-  -   -  n
UR                  U
5        M  UR                  S5        M     [        U5      [        U5      -  n[        S[        R                  " U5      5      $ )zCoverage scoring - measures query term presence across knowledge base

Returns a score between 0 and 1, where:
- 0 means no query terms found
- 1 means excellent coverage of all query terms
r   r   r   r   r   )r   r0   	_tokenizelowerr*   maxvaluesrZ   r,   mathlogappendsumlenminsqrt)rL   r^   query_termsterm_scoresmax_tftermtfdfdoc_coveragefreq_signal
term_scorer   s               r'   r   'StatisticalStrategy._calculate_coverageH  sF    {{e33q8nnU[[%6%6%899>9O9OU++2245UVD''++D!4B++//a8BAv!$9$99 JPRSdhhq2v.!f*1EEYZ *Q{1B-BC
"":.""3'  $ {#c+&66 3		(+,,r&   c                 6   [        UR                  5      S:  a  g/ n[        [        UR                  5      5       H  n[        US-   [        UR                  5      5       H  n[        U R	                  UR                  U   5      5      n[        U R	                  UR                  U   5      5      nU(       d  MZ  U(       d  Mc  [        XV-  5      [        XV-  5      -  nUR                  U5        M     M     U(       a  [        U5      [        U5      -  nU$ SnU$ )zQInformation overlap between pages - high overlap suggests coherent topic coverager?   r   r   r   )r  r   ranger.   _get_document_termsr   r  )	rL   r^   overlapsijterms_iterms_joverlapr   s	            r'   r   *StatisticalStrategy._calculate_consistencyq  s    u##$q( s5//01A1q5#e&:&:";<d66u7K7KA7NOPd66u7K7KA7NOP7ww!'"34s7;L7MMGOOG, = 2 h-#h-7K  Kr&   c                 $   UR                   (       d  g[        UR                   5      S:  a  gUR                   S   S:  a  UR                   S   OSnUR                   S   S:  a  UR                   S   OSnSX#-  -
  n[        S[        US5      5      $ )zIDiminishing returns indicator - are we still discovering new information?r   r?   r   r   r   )r1   r  r   r  )rL   r^   recent_rateinitial_rater   s        r'   r   )StatisticalStrategy._calculate_saturation  s    &&u&&'!+ 6;5L5LR5PST5Te--b1Z[5:5L5LQ5ORS5Su..q1YZ +45
3J,--r&   r   c                 \  #    / nUR                    H  nUR                  UR                  ;   a  M  U R                  XA5      nU R	                  XA5      nSnUR
                  U-  UR                  U-  -   UR                  U-  -   nUR                  XH45        M     UR                  S SS9  U$ 7f)z'Rank links by expected information gainr   c                     U S   $ Nr   r%   xs    r'   r(   0StatisticalStrategy.rank_links.<locals>.<lambda>  s    !r&   Tkeyreverse)
r   hrefr   _calculate_relevance_calculate_noveltyr   r   r   r   sort)	rL   r^   r   scored_linksrN   	relevancenovelty	authorityscores	            r'   r   StatisticalStrategy.rank_links  s     ''DyyE... 11$>I--d:GI ,,y8))G34++i78E .! (& 	nd;s   B*B,rN   c                    UR                   (       a  U(       d  gSR                  [        SUR                  =(       d    SUR                  =(       d    SUR
                  (       a,  UR
                  R                  S0 5      R                  SS5      OSUR
                  (       a,  UR
                  R                  S0 5      R                  SS5      OSUR
                  (       a,  UR
                  R                  S0 5      R                  SS5      OS/5      5      R                  5       nU(       d  gUR                  (       a  UR                  S	:  a  UR                  $ [        U R                  UR                   R                  5       5      5      n[        U R                  U5      5      nU(       d  g[        XE-  5      [        U5      -  nU$ )
z3BM25 relevance score between link preview and queryr    Nr   metatitledescriptionkeywordsr   )r   joinfiltertextr4  	head_datarZ   r   contextual_scorer.   r   r  )rL   rN   r^   	link_textr  
link_termsr  s          r'   r(  (StatisticalStrategy._calculate_relevance  sP   {{$ HHVDIIOJJ"?C~~DNNvr*..w;SUEI^^DNNvr*..}bAY[BF..DNNvr*..z2>VX+
   EG 	    T%:%:Q%>((( $..):):)<=>	23
k./#k2BBr&   c                    UR                   (       d  gSR                  [        SUR                  =(       d    SUR                  =(       d    SUR
                  (       a  UR
                  R                  SS5      OSUR
                  (       a  UR
                  R                  SS5      OSUR
                  (       a  UR
                  R                  SS5      OS/5      5      R                  5       n[        U R                  U5      5      nU(       d  g[        UR                  R                  5       5      nXE-
  nU(       a  [        U5      [        U5      -  nU$ S	nU$ )
z9Estimate how much new information this link might provider   r2  Nr   r4  r5  r6  r   r   )r   r7  r8  r9  r4  r:  rZ   r   r.   r   r*   keysr  )rL   rN   r^   r<  r=  existing_terms	new_termsr-  s           r'   r)  &StatisticalStrategy._calculate_novelty  s   ## HHVDIIOJJ"/3~~DNNw+259^^DNN}b126..DNNz2.b+
   EG 	 	23
 U3388:;/	6@#i.3z?2 GJr&   c                    SnUR                   (       d  gUR                   R                  5       nSU;   d  SU;   a  US-  nSU;   d  SU;   a  US-  nSU;   d  S	U;   a  US
-  nUR                  S5      (       a  US
-  nOUR                  S5      (       a  US-  nUR                  b  SU-  SUR                  -  -   n[	        US5      $ )zASimple authority score based on URL structure and link attributesr   r   z/docs/z/documentation/r   z/api/z/reference/z/guide/z
/tutorial/r   z.pdf)z.jpgz.pngz.gifr   r   r   )r'  r   endswithintrinsic_scorer  )rL   rN   r/  rh   s       r'   _calculate_authority(StatisticalStrategy._calculate_authority  s    yyiioo s?/36SLEc>]c1SLE|s2SLE <<SLE\\233SLE +%K#(<(<"<<E5#r&   c                 &  #    UR                   R                  SS5      nX2R                  :  a  g[        UR                  5      UR
                  :  a  gUR                  (       d  gUR                   R                  SS5      UR                  :  a  gg7f)r   r   r   Tr   F)r    rZ   r   r  r   r   r   r   )rL   r^   r   r   s       r'   r   StatisticalStrategy.should_stop  s      ]]&&|S9
444 u!!"f&6&66 "" ==\3/63N3NNs   BBr   Nc                   #    U GHV  n[        UR                  5      n UR                  R                  nU R                  UR                  5       5      n[        5       nU H+  nUR                  U==   S-  ss'   UR                  U5        M-     UR                  n	U HL  nXR                  U   ;  d  M  UR                  U==   S-  ss'   UR                  U   R                  U	5        MN     [        UR                  5      n
X-
  nUR                  R                  U5        U=R                  S-  sl        UR                   R                  UR                  5        GMY     g! [         a    [        SUR                   S35        Sn GNTf = f7f)r   zWarning: CrawlResult z has no markdown contentr   r   N)r  r*   rc   rd   AttributeErrorprintrh   r   r   r.   addr0   r/   r,   r1   r   r2   )rL   r^   r   resultold_term_countri   termsterm_setr  doc_idnew_term_countrB  s               r'   r    StatisticalStrategy.update_state$  s]    !F !7!78N //66  NN7==?3E uH&&t,1,T" 
 **F 99$??..t494..t488@ ! !!7!78N&7I##**95 !!Q&! $$VZZ0] " " -fjj\9QRSs/   FE!A;F4B-F!%F
F	F

Fr9  c                     [         R                  " SSU5      nUR                  5       nU Vs/ s H  n[        U5      S:  d  M  UPM     nnU$ s  snf )z%Simple tokenization - can be enhancedz[^\w\s]r2  r?   )resubsplitr  )rL   r9  tokensts       r'   r   StatisticalStrategy._tokenizeV  sK     vvj#t, $2Vs1vz!V2 3s   AAcrawl_resultc                 ~    UR                   R                  =(       d    SnU R                  UR                  5       5      $ )z!Extract terms from a crawl resultr   )rc   rd   r   r   )rL   r]  ri   s      r'   r  'StatisticalStrategy._get_document_termsa  s-    ''44:~~gmmo..r&   )r   r   r   )ry   rz   r{   r|   r   ru   r   r   r   r   r   r   r   r   r	   r   r   r(  r)  rG  r   r   r   r   rk   r   r  r}   r%   r&   r'   r   r   -  s4   ;

 u &'- '- '-RJ 5 6.: .% ."j . TRWX\^cXcRdMe 6 j U @t J 5 4 % :z > d *01
 01kAR 01W[ 01d	c 	d3i 	/ /S	 /r&   r   c                      \ rS rSrSrS'S\S\\\4   S\\\4   4S jjr	S\
\   4S	 jrS\
\   4S
 jrS\\   S\4S jrS\S\S\4S jrS\S\S\4S jrS(S\S\S\4S jjrS)S\S\4S jjrS*S\S\\   4S jjrS\S\S\\\\4      4S jrS\\   S\\\\4      S\S\\\\4      4S jrS\S\4S jrS\S\S\\\\4      4S  jrS\S\4S! jrS\S\S\ 4S" jr!S\S\4S# jr"S\S$\\#   SS4S% jr$S&r%g)+EmbeddingStrategyig  z?Embedding-based adaptive crawling using semantic space coverageNr8   
llm_configr   c                     U=(       d    SU l         X l        X0l        0 U l        0 U l        SU l        S U l        S U l        S U l        SU l	        g )Nr   Fr   )
r8   rb  r   _embedding_cache_link_embedding_cache_validation_passed_distance_matrix_cache_kb_embeddings_hash_validation_embeddings_cache_kb_similarity_threshold)rL   r8   rb  r   s       r'   ru   EmbeddingStrategy.__init__j  sU    .Z2Z$ 0 "%'""' '+##' ,0)(,%r&   rU   c                     [        U S5      (       a0  U R                  (       a  U R                  R                  nU(       a  U$ g)z?Get embedding LLM config as dict, or None for local embeddings.r   N)rj   r   r   rL   config_dicts     r'   _get_embedding_llm_config_dict0EmbeddingStrategy._get_embedding_llm_config_dictx  s2    4""t{{++@@K"" r&   c                    U R                   bE  [        U R                   [        5      (       a  U R                   $ U R                   R                  5       $ [	        U S5      (       a0  U R
                  (       a  U R
                  R                  nU(       a  U$ U R                  bE  [        U R                  [        5      (       a  U R                  $ U R                  R                  5       $ g)a  Get query LLM config as dict for chat completion calls.

Fallback chain:
1. self.query_llm_config (explicit query config on strategy)
2. self.config._query_llm_config_dict (from AdaptiveConfig)
3. self.llm_config (legacy: single config for both)
4. None (caller uses hardcoded defaults)
Nr   )r   r   rF   r   rj   r   r   rb  rm  s     r'   _get_query_llm_config_dict,EmbeddingStrategy._get_query_llm_config_dict  s       ,$//66,,,((0022 4""t{{++<<K"" ??&$//400&??**,, r&   textsc                 r   #    SSK Jn  U R                  5       nU" UUU R                  5      I Sh  vN $  N7f)z&Get embeddings using configured methodr   get_text_embeddingsN)utilsrw  ro  r8   )rL   rt  rw  r   s       r'   _get_embeddings!EmbeddingStrategy._get_embeddings  s=     .#BBD(   
 
 	
 
s   .757r4   r3   c                    Ub  [        U5      S:X  a  g[        UR                  5      S:X  a  UR                  SS5      n[        UR                  5      S:X  a  UR                  SS5      nU[        R                  R                  USSS9-  nU[        R                  R                  USSS9-  n[        R                  " X4R                  5      nSU-
  nU$ )z3Compute distance matrix using vectorized operationsNr   r   r  Taxiskeepdims)r  shapereshaper[   linalgnormdotT)rL   r4   r3   
query_normkb_normsimilarity_matrixdistance_matrixs          r'   _compute_distance_matrix*EmbeddingStrategy._compute_distance_matrix  s      C$6!$; %%&!+/772>}""#q()11!R8M &		7GaZ^(__
"))..QQU."VV FF:yy9 //r&   c                     Ub  [        U5      S:X  a  gUb  [        UR                  5       5      OSnU R                  b  X0R                  :w  a  U R                  X5      U l        X0l        U R                  $ )z Get distance matrix with cachingNr   )r  hashtobytesrg  rh  r  )rL   r4   r3   kb_hashs       r'   _get_cached_distance_matrix-EmbeddingStrategy._get_cached_distance_matrix  su      C$6!$; 4A3L$},,./RV''////*.*G*GHX*hD''.$***r&   r   n_syntheticc                   #    SSK Jn  [        US-  5      nSU SU S3nU R                  5       nU(       a  UR	                  SS5      OSnU(       a  UR	                  S	5      OS
nU(       a  UR	                  S5      OS
n	U" UUUSU	U(       a  UR	                  SS5      OSU(       a  UR	                  SS5      OSU(       a  UR	                  SS5      OSS9n
[
        R                  " U
R                  S   R                  R                  5      nSS
K
nUS   R                  5       nUR                  U5        [        S[        [        U5      S-  5      5      nX* S
 nU/US
U*  -   nU R                  U5      I S
h  vN nXl        UU4$  N7f)zJGenerate a point cloud representing the semantic neighborhood of the queryr   )perform_completion_with_backoffg?z	Generate z; variations of this query that explore different aspects: 'z'
        
        These should be queries a user might ask when looking for similar information.
        Include different phrasings, related concepts, and specific aspects.
        
        Return as a JSON array of strings.providerzopenai/gpt-4o-mini	api_tokenNbase_urlTbackoff_base_delayr?   backoff_max_attemptsr   backoff_exponential_factor)r  prompt_with_variablesr  json_responser  
base_delaymax_attemptsexponential_factorr   queriesr   )rx  r  r$   rr  rZ   rJ   loadschoicesmessageri   randomcopyshuffler   r  ry  _validation_queries)rL   r   r  r  n_totalpromptllm_config_dictr  r  r  response
variationsr  other_queriesn_validationval_queriestrain_queriestrain_embeddingss                     r'   map_query_semantic_space*EmbeddingStrategy.map_query_semantic_space  s    : kC'( wi'bchbi j+ . 99;L[?&&z3GHau8GO''4T	6E?&&z242"(GV**+?C\]KZ,,-CQG`aWf223OQRSlm	
 ZZ 0 0 3 ; ; C CD
4 	 #9-224}% 1c#m"4s":;<#MN3-,"?? "&!5!5m!DD $/ .. Es   E8F:F	;Fquery_pointsalphac                 Z    [        U5      S:  a  g[        R                  " U5      n[        R                  " USS9[        R                  " USS9U[        R
                  " [        R                  R                  U[        R                  " USS9-
  SS95      S.nU$ ! [         a     gf = f)zEFind the minimal shape that covers all query points using alpha shaper   Nr   r}  r   )centerstdpointsradius)	r  r[   r\   meanr  r   r  r  	Exception)rL   r  r  r   s       r'   compute_coverage_shape(EmbeddingStrategy.compute_coverage_shape$  s    	 < 1$
 88L1L '',Q7vvl3&&&rww|Z[?\0\cd!ef	H O 		s   B B
B 
B*)B*	n_samplesc                     Uc  / $ / $ )z*Sample points from the boundary of a shaper%   )rL   r  r  s      r'   _sample_boundary_points)EmbeddingStrategy._sample_boundary_points=  s     =I 	r&   c                 J   / nUb  [        U5      S:X  a  U H  nUR                  US45        M     U$ U R                  X!5      nUc  U H  nUR                  US45        M     U$ [        R                  " USS9n[        U5       H  u  ptUR                  XFU   45        M     U$ )zLCalculate gap distances for all query variations using vectorized operationsr   r   r   r  )r  r   r  r[   r  	enumerate)rL   r3   r4   gapsq_embr  min_distancesr  s           r'   find_coverage_gaps$EmbeddingStrategy.find_coverage_gapsI  s      C$6!$;)UCL) *K ::;K[")UCL) *K Q7 ""23HAKKa 012 4 r&   candidate_linksr  c                   ^$#    SSK JnJnJn  SSKn/ n/ n	/ n
0 nU GHf  nSR                  [        SUR                  =(       d    SUR                  =(       d    S[        US5      (       a-  UR                  (       a  UR                  R                  SS5      OSUR                  (       a,  UR                  R                  S0 5      R                  SS5      OS/5      5      nUR                  5       (       d  M  UR                  UR                   S	U 3R!                  5       5      R#                  5       nXR$                  ;   a  U R$                  U   XR                  '   GMD  U	R'                  U5        U
R'                  U5        GMi     U
(       a  U R)                  5       nU" XU R*                  5      I Sh  vN n[-        XU5       H`  u  nnnUR                  UR                   S	U 3R!                  5       5      R#                  5       nUU R$                  U'   UXR                  '   Mb     [        U S
5      (       a  U R.                  R0                  OSm$U GH  nUR                  U;  a  M  XR                     nU(       d  SnGO_SnSnU H3  u  nnUT$:  d  M  U" UU5      nUU:  d  M  UU-
  nUS-  nUU-  nUS-  nM5     [3        U$4S jU 5       5      nUS:  a  UU-  nOSnUb  [5        U5      S:  a  U[6        R8                  R;                  U5      -  nU[6        R8                  R;                  USSS9-  n[6        R<                  " UU5      n [6        R>                  " U 5      n![        U S
5      (       a  U R.                  R@                  OSn"U!U":  a	  U!U"-
  S-  n#OSn#OSn#USU#-
  -  n[        US5      (       a&  URB                  (       a  US-  URB                  S-  -   nUR'                  UU45        GM     [E        US SS9$  GNR7f)z0Select links that most efficiently fill the gapsr   )cosine_distancecosine_similarityrw  r   Nr2  r   r3  r5  :r   r   r   r?   c              3   <   >#    U  H  u  pUT:  d  M  S v   M     g7f)r   Nr%   ).0_r_   coverage_radiuss      r'   	<genexpr>?EmbeddingStrategy.select_links_for_expansion.<locals>.<genexpr>  s     'Rddaa/>Qds   	Tr|  r   r;  r   c                     U S   $ r   r%   r!  s    r'   r(   >EmbeddingStrategy.select_links_for_expansion.<locals>.<lambda>  s    !A$r&   r$  )#rx  r  r  rw  hashlibr7  r8  r9  r4  rj   r3  rZ   r:  stripmd5r'  encode	hexdigestre  r   ro  r8   zipr   r   r  r  r[   r  r  r  r   r   r;  sorted)%rL   r  r  r3   r  r  rw  r  r+  links_to_embedtexts_to_embedlink_embeddings_maprN   r<  	cache_keyr   new_embeddingsr9  	embeddinglink_embeddingr/  gaps_helpedtotal_improvement	gap_pointgap_distancenew_distanceimprovementscaled_improvementgaps_needing_helpgap_reduction_score	link_normr  similaritiesmax_similarityoverlap_thresholdoverlap_penaltyr  s%                                       @r'   select_links_for_expansion,EmbeddingStrategy.select_links_for_expansiong  s     	SR  #D		R

 b4;D&4I4Idii		mR0]_IM""62.22="E]_	/ " I ??$$  tyyk9+$>$E$E$GHRRTI 666151K1KI1V#II.%%d+%%i0) $. #'#F#F#H #6~]a]q]q#rrN *-^^)\%dI#KK499+Qtf(=(D(D(FGQQS	8A**951:#II. *] DK4QYCZCZ$++??`c $Dyy 330;N  $%!/3+I|#o5'6~y'Q',6*6*EK1<q.-1CC-'1,K 04 %('Rd'R$R!$q(*;>O*O'*+' !,]1Ca1G .1O OI+biinn]QR]an.bbG $&66'9#=L%'VVL%9N T[[_aiSjSj(O(Opt%%(99+9<M+MQR*R*+&'O ,q?/BC 4!3449N9N!CK$*?*?#*EEEu.u $x lEEQ ss    F1P	4P5CP	P	(EP	r^   c                   #    UR                   b  UR                  c  g[        UR                   5      S:X  d  [        UR                  5      S:X  a  g[        R                  " UR                  [        R
                  S9n[        R                  " UR                   [        R
                  S9nU[        R                  R                  USSS9S-   -  nU[        R                  R                  USSS9S-   -  nX#R                  -  R                  SS	9n[        U R                  S
S5      nUb  [        XE:  R                  5       5      O[        UR                  5       5      nXaR                  S'   [        UR                  5       5      UR                  S'   [        [        R                  " U5      5      UR                  S'   U$ 7f)u&   Coverage-based learning score (0–1).Nr   r   )dtyper   Tr|  g:0yE>r  coverage_taucoverage_scoreavg_best_similaritymedian_best_similarity)r3   r4   r  r[   asarrayfloat32r  r  r  r   getattrr   r   r  r    median)rL   r^   QDbesttaur/  s          r'   r   &EmbeddingStrategy.calculate_confidence  s_     &%*@*@*Hu""#q(C0F0F,G1,L JJu--RZZ@JJu**"**=	RYY^^AA^5<<	RYY^^AA^5<< CC}}!}$ dkk>48/2t{((*+E$))+DV +0&'/4TYY[/A+,27		$2H./s   F>G r   c                   #    X l         [        5       n/ nUR                   H]  nUR                  UR                  ;  d  M  UR                  U;  d  M1  UR                  U5        UR                  UR                  5        M_     U(       d  / $ U R                  UR                  UR                  5      nU Vs/ s H  owS   R                  5       US   4PM     snUl        U R                  UUUR                  5      I Sh  vN $ s  snf  N	7f)z!Main entry point for link rankingr   r   N)r   r.   r   r'  r   r   rN  r  r3   r4   rH   r7   r  )rL   r^   r   	seen_urlsuncrawled_linksrN   r  gs           r'   r   EmbeddingStrategy.rank_links@  s       E	''Dyy 2 22tyy	7Q&&t,dii( (
 I &&""
 ?CCd!qt4dC 44
 
 	
 D
s*   :D
 D
A#D
5!D(D
>D?
D
c                   #    [        U S5      (       a  U R                  (       d  UR                  R                  SS5      $ U R                  c(  U R                  U R                  5      I Sh  vN U l        U R                  nUR                  b  [        UR                  5      S:X  a  gU R                  X!R                  5      nUc  g[        R                  " USS9nSU-
  n[        R                  " U5      nXaR                  S	'   U$  N7f)
z5Validate coverage using held-out queries with cachingr  r   r   Nr   r   r  r   validation_confidence)rj   r  r    rZ   ri  ry  r3   r  r  r[   r  r  )rL   r^   val_embeddingsr  r  scoresr
  s          r'   validate_coverage#EmbeddingStrategy.validate_coverage_  s     t2334;S;S==$$\377
 ,,46:6J6J4KcKc6d0dD-:: &#e.A.A*Ba*G 77H[H[\" Q7}$ !#1F-.$$3 1es   A*D
,D-BD
c                   #    UR                   R                  SS5      n[        US5      (       a  UR                  OSnX4:  a8  [	        UR
                  5      S:  a  SUR                   S'   SUR                   S	'   g[	        UR
                  5      UR                  :  d  UR                  (       d  g[        US
5      (       d  / Ul        UR                  R                  U5        [	        UR                  5      S:  a  g[        [        UR                  SS UR                  SS 5      5      n[        S U 5       5      [	        U5      -  nXaR                   S'   [        U S5      (       a  U R                  R                  U-  OSU-  nXg:  am  U R                  U5      I Sh  vN n[        U S5      (       a  U R                  R                   OSn	X:  a  SUR                   S'   SU l        gSUR                   S'   g NY7f)z(Stop based on learning curve convergencer   r   r   r   r   !below_minimum_relevance_thresholdstopped_reasonTis_irrelevantconfidence_historyr?   FNr  r   c              3   @   #    U  H  u  p[        X!-
  5      v   M     g 7fr"   )r   )r  abs      r'   r  0EmbeddingStrategy.should_stop.<locals>.<genexpr>  s     G5FTQc!%jj5Fs   avg_improvementr   r   converged_validatedlow_validation)r    rZ   rj   r   r  r   r   r   r  r   rC   r  r  r   r   r  r   rf  )
rL   r^   r   r   min_confidence_thresholdimprovement_diffsr  min_relative_improvement	val_scorevalidation_mins
             r'   r   EmbeddingStrategy.should_stop  s    ]]&&|S9
 QXX^  aE  QF  QF6#L#L  LO 0S9K9K5Lq5P.QEMM*+-1EMM/* u!!"f&6&66e>Q>Q u233')E$  ''
3 u''(1, U%=%=cr%BED\D\]^]_D`!ab G5FGG#N_J``+:'(bijnpxbyby4;;#Q#QT^#^  @C  FP  @P 5"44U;;I LSSWYaKbKbT[[GGhkN )2G./*.'2B./  <s   FG< G:!AG<c                 6   UR                   R                  SS5      nUR                   R                  SS5      n[        U S5      (       a  U R                  R                  OSn[        U S5      (       a  U R                  R
                  OSn[        U S5      (       a  U R                  R                  OSn[        U S5      (       a  U R                  R                  OSnU R                  (       a&  X4:  a!  US:  a  UnU$ US:  a  UnU$ XRS-
  U-  -   n U$ US	-  nU$ )
z4Calculate quality-based confidence score for displaylearning_scorer   r
  r   r   r   r   r   r   )	r    rZ   rj   r   r   r   r   r   rf  )	rL   r^   r"  validation_scorer  quality_minquality_maxscale_factorr   s	            r'   get_quality_confidence(EmbeddingStrategy.get_quality_confidence  s   **+;SA ==,,-DcJ HOtU]G^G^CCdgFMdT\F]F]dkkBBcfFMdT\F]F]dkkBBcgELTS[E\E\t{{AAbg""'7'H #(
   #%(
  )S,@L+PP

  (#-Jr&   r   c                 @  #    SSK Jn  / n/ nU Hk  n[        US5      (       a'  UR                  (       a  UR                  R                  OSnU(       d  MF  UR                  USS 5        UR                  U5        Mm     U(       d  gU R                  5       nU" XHU R                  5      I Sh  vN n	UR                  c%  Xl        [        [        [        U	5      5      5      n
GO/ n/ n
[        U	5       H  u  pU[        R                  R                  U5      -  nUR                  [        R                  R                  UR                  SSS9-  n[        R                   " X5      n[        R"                  " U5      U R$                  :  d  M  UR                  U5        U
R                  U5        M     U(       a;  [        R&                  " UR                  [        R(                  " U5      /5      Ul        U
 H+  nUR*                  R                  UU   R,                  5        M-     SU l        SU l        [        US	5      (       aW  UR2                  bI  U R5                  UR2                  [        U S
5      (       a  U R6                  R8                  OS5      Ul        ggg GN7f)z9Update embeddings and coverage metrics with deduplicationr   rv  rc   r   Ni  Tr|  r4   r   r   )rx  rw  rj   rc   rd   r   ro  r8   r3   rC   r  r  r  r[   r  r  r  r   rj  vstackr\   r2   rh   rh  rg  r4   r  r   r   r6   )rL   r^   r   rw  	new_textsvalid_resultsrO  ri   r   r  deduplicated_indicesdeduplicated_embeddingsr  new_embnew_emb_normalizedkb_normalizedr  idxs                     r'   r   EmbeddingStrategy.update_state  sM    . 	!F6=fj6Q6QV\VeVefoo22kmGw  $0$$V,	 "   $BBD29TXThThii &"0#'c..A(B#C  ')##% '7
%,ryy~~g/F%F" % 3 3biinnUEXEX_`kon6p p!vvmH 66,'$*G*GG+227;(//2 8 '&(ii1D1DbhhOfFg0h&i# (C$$]3%7%;%;< ( $( &*# 5,--%2H2H2T#'#>#>u?U?Ux  AE  GO  yP  yPW[WbWbWtWt  VY  $ZE  3U-I js!   AJAJ,J-CJDJ)rg  rd  rh  rj  re  ri  rf  r  r   r8   rb  r   NNN)r   )r   )r   )&ry   rz   r{   r|   r   rk   r   r   r   ru   r   ro  rr  r   r
   ry  r  r  r$   r  r   r  r  r	   r  r   r  r   r   r   r   r  r   r   r'  r   r   r}   r%   r&   r'   ra  ra  g  s&   I- -iQUo@V -qv  xA  CG  xG  rH - HTN <
49 
 
 S UX 4+C +PS +X[ +$L/C L/c L/SV L/\3 u 2
 
T#Y 
 s tTYZ]_dZdTeOf <sFdsF 5e$%sF 	sF
 
eD%K 	!sFj
 u H
j 
. 
TRWX\^cXcRdMe 
>"%Z "%E "%H1z 1> 1d 1fJ 5 68Z
 8ZkAR 8ZW[ 8Zr&   ra  c            	          \ rS rSrSr   S#S\\   S\\   S\\   4S jjr	S\
S	\4S
 jr S$S\
S\
S\\
   S	\4S jjrS\
S\
S	\\   4S jrS\\\\4      S\
S	\\   4S jr\S	\4S j5       r\S	\\
\4   4S j5       r\S	\4S j5       rS%S\S	S4S jjrS	\
4S jrS&S\\
\4   S\
S	S4S jjr S	\\
\4   4S jr!S&S\\
\4   S\
S	S4S jjr"S\\
\4   4S jr#S'S \$S	\\\
\4      4S! jjr%S"r&g)(AdaptiveCrawleri  z<Main adaptive crawler that orchestrates the crawling processNcrawlerr   r   c                    Xl         U=(       d
    [        5       U l        U R                  R                  5         U(       a  X0l        O*U R                  U R                  R                  5      U l        S U l        US L U l        g r"   )r7  r   r   r   r   _create_strategyr^   _owns_crawler)rL   r7  r   r   s       r'   ru   AdaptiveCrawler.__init__  se     0 0 $M 11$++2F2FGDM ,0
 %_r&   strategy_namerU   c                     US:X  a
  [        5       $ US:X  aZ  [        U R                  R                  U R                  R                  U R                  R
                  S9nU R                  Ul        U$ [        SU 35      e)z&Create strategy instance based on namer   r  )r8   rb  r   zUnknown strategy: )r   ra  r   r8   r   r   
ValueError)rL   r<  r   s      r'   r9   AdaptiveCrawler._create_strategy#  sq    M)&((k)( $ ; ;;;;;!%!=!=H
 #kkHOO1-ABBr&   	start_urlr   resume_fromc           	        #    U(       a+  [         R                  U5      U l        X R                  l        O[        [	        5       / / U0 S9U l        U R
                  (       d1  [        5       U l        U R
                  R                  5       I Sh  vN   U R                  U R                  l        [        U R                  [        5      (       a  U(       d  U R                  R                  UU R                  R                  5      I Sh  vN u  pEX@R                  l        USS U R                  l        U R                  R                   U R                  l         XR                  R"                  ;  Ga  U R%                  X5      I Sh  vN nU(       Gar  ['        US5      (       Ga`  UR(                  (       GaN  U R                  R*                  R-                  U5        U R                  R"                  R/                  U5        ['        US5      (       a  UR0                  (       a  [        UR0                  [2        5      (       a[  UR0                  R5                  S/ 5       Vs/ s H  n[7        S0 UD6PM     nnU R                  R8                  R;                  U5        O9U R                  R8                  R;                  UR0                  R<                  5        U R                  R?                  U R                  U/5      I Sh  vN   Sn	XR                  R@                  :  Gax  U R                  RC                  U R                  5      I Sh  vN n
XR                  RD                  S'   U R                  RG                  U R                  U R                  5      I Sh  vN (       a  GOU R                  RI                  U R                  U R                  5      I Sh  vN nU(       d  GOUS   S   U R                  RJ                  :  a  GOUSU R                  RL                    VVs/ s H.  u  p|URN                  U R                  R"                  ;  d  M+  X|4PM0     nnnU(       d  GO4U RQ                  X5      I Sh  vN nU(       Ga  U R                  R*                  R;                  U5        [S        X5       GH*  u  nu  pU(       d  M  U R                  R"                  R/                  URN                  5        ['        US5      (       d  MT  UR0                  (       d  Mg  / n[        UR0                  [2        5      (       a8  UR0                  R5                  S/ 5       Vs/ s H  n[7        S0 UD6PM     nnUnOUR0                  R<                  nU HN  nURN                  U R                  R"                  ;  d  M)  U R                  R8                  R-                  U5        MP     GM-     U R                  R?                  U R                  U5      I Sh  vN   U	S-  n	U R                  RT                  (       aJ  U R                  RV                  (       a/  U R                  RY                  U R                  RV                  5        XR                  R@                  :  a  GMx  U R                  RC                  U R                  5      I Sh  vN n[        U R                  [        5      (       a=  U R                  R[                  U R                  5      U R                  RD                  S'   OUU R                  RD                  S'   []        U R                  R"                  5      U R                  RD                  S	'   XR                  RD                  S
'   U R                  RT                  (       aJ  U R                  RV                  (       a/  U R                  RY                  U R                  RV                  5        U R                  U R^                  (       a8  U R
                  (       a&  U R
                  Ra                  SSS5      I Sh  vN   $ $ $  GN GN? GNs  snf  GNQ GN GN GNs  snnf  GNs  snf  GN] GN N5! U R^                  (       a9  U R
                  (       a'  U R
                  Ra                  SSS5      I Sh  vN    f f f = f7f)z&Main entry point for adaptive crawling)r   r   r   r   r    Nr   successre   internalr   r   pages_crawleddepth_reachedr%   )1r   rX   r^   r   r.   r7  r   
__aenter__r   r   r   ra  r  r   r4   r5   r8   r   _crawl_with_previewrj   rC  r   r   rN  re   rF   rZ   r   r   extendrD  r   r   r   r    r   r   r   r   r'  _crawl_batchr  r   r   rS   r'  r  r:  	__aexit__)rL   r@  r   rA  r4   r5   rO  rN   internal_linksdepthr   ranked_linksr/  to_crawlr   r  	new_links	link_datanew_linkr"  s                       r'   digestAdaptiveCrawler.digest2  sI     #5DJ$JJ# U! DJ ||*,DL,,))+++#{{ dmm%6777;}}7]7]..8 2. +;JJ'*:12*>DJJ')-)F)FDJJ&j	?

 7 77#77	IIgfi88V^^^JJ--44V<JJ++//	:vw//FLL%fllD99GM||GWGWXbdfGg-hGgtdlTlGgN-h JJ44;;NK !JJ44;;FLL<Q<QR --44TZZ&JJJ E++///#'==#E#Edjj#QQ
3=

""<0 224::t{{KKK &*]]%=%=djj$++%VV#  ?1%(F(FF >JJb4;;KbKb=c H=ckd"iitzz/F/FF *TM=c H   %)$5$5h$FFJJ--44[A .1-G)	!6 JJ3377		B&vw77FLLL,.	#-fllD#A#AY_YeYeYiYijtvxYy5zYyId6GY6GYyN5z0>I 170E0EI 1:H'/}}DJJ<S<S'S(,

(@(@(G(G(Q 1: .H( --44TZZMMM
 ;;))dkk.D.DJJOODKK$:$:;y ++///~ $(==#E#Edjj#QQN $--):;;37==3W3WX\XbXb3c

""<0 4B

""<025djj6M6M2NDJJ/27JJ/ {{%%$++*@*@

 6 67:: !!dllll,,T4>>> '3!q ,2 J .i K R L  WH G 6{ N R* ? !!dllll,,T4>>> '3!s@  B
c aA8c aAc .a1 aCa1 a
3Ba1 >a?Aa1 aAa1 a>a1 aAa1 "*aa a1 7a!8A
a1 >a1 a1 ?a1 a$/Aa1 5Aa1 a)Ba1 (a1 ?a, D7a1 7Ac 9a/:c c a1 
a1 a1 a1 a1 a1 $a1 ,a1 /c 1Ab=4b75b==c rh   c                 4  #    [        [        SSUSU R                  R                  SSS9SS9n U R                  R                  XS9I Sh  vN n[        US	5      (       a   UR                  (       a  UR                  S
   n[        US5      (       aU  UR                  (       aD  UR                  S    Vs/ s H  oUR                  S5      (       d  M  UPM     snUR                  S'   U$  Ns  snf ! [         a  n[        SU SU 35         SnAgSnAff = f7f)z%Crawl a URL with link preview enabledTFr   2   )include_internalinclude_externalr   concurrencytimeout	max_linksverbose)link_preview_configscore_links)rh   r   N_resultsr   re   rD  r:  zError crawling z: )r   r   r   r   r7  arunrj   r_  re   rZ   r  rM  )rL   rh   r   r   rO  rN   es          r'   rH  #AdaptiveCrawler._crawl_with_preview  s     ! 1!%!&88! 
	<<,,,DDFvz**v+ vw''FLL=C\\*=U+o=UTYaYabmYnD=U+oZ( M E ,p
  	OC51#./	sY   ,DC0 C)A(C0 4C+C+C0 (D)C0 +C0 0
D:DDDDlinks_with_scoresc                   #    / nU H2  u  pEU R                  UR                  U5      nUR                  U5        M4     [        R                  " USS06I Sh  vN n/ nU H  n	[        U	[        5      (       ab  [        U	S5      (       a$  U	R                  (       a  UR                  U	5        MM  [        S[        U	S5      (       a  U	R                  OS 35        Mz  [        U	[        5      (       d  M  [        SU	 35        M     U$  N7f)	zCrawl multiple URLs in parallelreturn_exceptionsTNrC  zSkipping failed crawl: rh   unknownzError in batch crawl: )rH  r'  r   asynciogatherr   r   rj   rC  rM  rh   r  )
rL   rc  r   tasksrN   r/  taskresultsr,  rO  s
             r'   rJ  AdaptiveCrawler._crawl_batch  s     ,KD++DIIu=DLL -  FFF F&+..69--&..!((03'&RWBXBXFJJ^g3hijFI...vh78   Gs   ADDBD0Dc                 r    U R                   (       a&  U R                   R                  R                  SS5      $ g)zCurrent confidence levelr   r   )r^   r    rZ   r   s    r'   r   AdaptiveCrawler.confidence  s+     ::::%%)),<<r&   c                 |   U R                   (       d  0 $ [        S U R                   R                   5       5      n[        U R                   R                  5      U[        U R                   R
                  5      [        U R                   R
                  R                  5       5      [        U R                   R                  5      U R                  U R                   R                  R                  SS5      U R                   R                  R                  SS5      U R                   R                  R                  SS5      S.	$ )zDetailed coverage statisticsc              3   r   #    U  H-  n[        UR                  R                  =(       d    S 5      v   M/     g7f)r   N)r  rc   rd   )r  rO  s     r'   r  1AdaptiveCrawler.coverage_stats.<locals>.<genexpr>  s.      #
3 ,,2333s   57r   r   r   r   )	rE  total_content_lengthunique_termstotal_termsr   r   r   r   r   )r^   r  r   r  r   r*   r   r   r   r    rZ   )rL   rr  s     r'   coverage_statsAdaptiveCrawler.coverage_stats  s     zzI" #
**33#
  
 !!8!89$8

 ; ;<tzz::AACD !9!9://

**..z3?::--11-E**,,00sC

 
	
r&   c                     [        U R                  [        5      (       a  U R                  R                  $ U R                  U R
                  R                  :  $ )z(Check if current knowledge is sufficient)r   r   ra  rf  r   r   r   r   s    r'   is_sufficientAdaptiveCrawler.is_sufficient  sA     dmm%677==333 ??dkk&F&FFFr&   detailedc                   ^  T R                   (       d  [        S5        g SSKJn  SSKJn  U" 5       nSnU(       Gd4  U(       Ga,  W" ST R                   R                   S	3S
9nUR                  SSSS9  UR                  SSS9  T R                  nUR                  S[        UR                  SS5      5      5        UR                  S[        UR                  SS5      5      5        UR                  S[        UR                  SS5      5      5        UR                  SUR                  SS5      S S35        UR                  S[        UR                  SS5      5      5        UR                  SS5        [        T R                  [        5      (       Ga  UR                  SUR                  SS5      S  5        UR                  S!T R                   R                   R                  S"S5      S# 5        UR                  S$T R                   R                   R                  S%S5      S& 5        UR                  S'T R                   R                   R                  S(S5      S  5        UR                  SS5        UR                  S)T R"                  (       a  S*OS+5        OUR                  SUR                  SS5      S  5        UR                  S,UR                  S-S5      S  5        UR                  S.UR                  S/S5      S  5        UR                  S0UR                  S1S5      S  5        UR                  SS5        UR                  S)T R"                  (       a  S2OS+5        WR                  U5        g[        S35        [        S4T R                   R                   S	35        [        S55        [        S65        [        S7[%        T R                   R&                  5       35        [        S8[%        T R                   R(                  5       35        [        S9T R                   R*                   35        [-        U 4S: jT R                   R.                   5       5      n[-        T R                   R0                  R3                  5       5      n	[%        T R                   R0                  5      n
[        S;5        [        S<US S=35        [        S>U	S 35        [        S?U
S 35        U	S:  a  [        S@X-  S  35        [        T R                  [        5      (       Ga  [        SA5        [        SBT R                   R                   R                  S"S5      S# 35        [        SCT R                   R                   R                  S%S5      S& 35        [        SDT R                   R                   R                  SES5      S& 35        [        SF5        T R"                  (       aC  U(       a!  WR                  SGT R4                  S  SH35        O][        SGT R4                  S  SI35        OBU(       a!  WR                  SGT R4                  S  SJ35        O[        SGT R4                  S  SK35        [        SLT R                   R                   R                  SMS5      S  35        [        SNT R                   R                   R                  S(S5      S  35        GO[        SO5        T R                  R7                  T R                   R                  R9                  5       5      nU GH#  nT R                   R0                  R                  US5      nT R                   R:                  R                  US5      nUS:  a  U(       aO  WR                  SPU SQU SRT R                   R*                   SSUT R                   R*                  -  ST SUU SV35        M  [        SPU SQU SRT R                   R*                   SWUT R                   R*                  -  ST SXU SV35        M  U(       a  WR                  SPU SY35        GM  [        SPU SZ35        GM&     [        SF5        T R"                  (       a  S[OS\nU(       a8  T R"                  (       a  S]OS^nWR                  SGT R4                  S  S_U 35        O[        SGT R4                  S  S_U 35        [        S`T R                   R                   R                  S-S5      S  35        [        SaT R                   R                   R                  S/S5      S  35        [        SbT R                   R                   R                  S1S5      S  35        T R                   R<                  (       a  [-        T R                   R<                  5      [%        T R                   R<                  5      -  n[        Sc5        [        SdUS& 35        [        SeT R                   R                   R                  S1S5      S  35        U(       Ga  [        Sf5        U(       a  WR                  Sg5        O[        Sh5        [        Si5        [        Sj5        [?        T R                   R0                  RA                  5       Sk SSl9SSm n[C        USn5       Hp  u  nu  nnT R                   R:                  R                  US5      nU(       a!  WR                  SoUSp SqU SrU SsU St3	5        MW  [        SoUSp SuU SvU SsU St3	5        Mr     [        Sw[%        T R                   R&                  5       Sx35        [C        T R                   RD                  Sn5       H  u  nnU[%        T R                   R<                  5      ::  a  T R                   R<                  USn-
     OSnU(       a/  WR                  SoU SyU Sz35        UR                  S{U S|35        M}  [        SoU S}U 35        [        S~U S35        M     [        S5        0 nT R                   R:                  R3                  5        H  nUR                  US5      Sn-   UU'   M     [?        URG                  5       5       H  nUU   n[        SU SU S35        M     T R                   RH                  (       Gap  [        S5        [        ST R                   RH                   35        [        S[%        T R                   RJ                  5       35        T R                   RL                  b-  [        ST R                   RL                  RN                   35        O[        S5        [        S[%        T R                   RP                  5       35        [        ST R4                  S  35        T R                   RJ                  (       aj  [        S5        [C        T R                   RJ                  SS Sn5       H8  u  nnU(       a  WR                  SoU SU S35        M'  [        SoU S}U 35        M:     [        S35        g! [         a    Sn GNf = f)zPrint comprehensive statistics about the knowledge base

Args:
    detailed: If True, show detailed statistics including top terms
zNo crawling state available.Nr   )Console)TableTFzAdaptive Crawl Stats - Query: '')r4  Metriccyan)styleno_wrapValuemagenta)r  zPages CrawledrE  zUnique Termsrs  zTotal Termsrt  zContent Lengthrr  ,z charszPending Linksr   r   
Confidencer   z.2%zAvg Min Distanceavg_min_distancez.3fzAvg Close Neighborsavg_close_neighborsz.1fzValidation Scorer
  zIs Sufficient?z[green]Yes (Validated)[/green]z[red]No[/red]Coverager   Consistencyr   
Saturationr   z[green]Yes[/green]zQ
================================================================================z$Adaptive Crawl Statistics - Query: 'zP================================================================================z
[*] Basic Statistics:z  Pages Crawled: z  Pending Links: z  Total Documents: c              3   X   >#    U  H  n[        TR                  U5      5      v   M!     g 7fr"   )r  _get_content_from_result)r  rO  rL   s     r'   r  .AdaptiveCrawler.print_stats.<locals>.<genexpr>d  s+      '7F D11&9::7s   '*z
[*] Content Statistics:z  Total Content: z charactersz  Total Words: z  Unique Terms: z  Vocabulary Richness: z 
[*] Semantic Coverage Analysis:z  Average Min Distance: z  Avg Close Neighbors (< 0.3): z$  Avg Very Close Neighbors (< 0.2): avg_very_close_neighborsz
[*] Confidence Metrics:z  Overall Confidence: z [green][VALIDATED][/green]z [VALIDATED]z [red][NOT VALIDATED][/red]z [NOT VALIDATED]z  Learning Score: r"  z  Validation Score: z
[*] Query Coverage:z  'z': found in /z docs ([green]z.0%z[/green]), z occurrencesz docs (z), z': [red][X] not found[/red]z': [X] not foundz[OK]z[!!]z[green][OK][/green]z[red][!!][/red]r2  z  Coverage Score: z  Consistency Score: z  Saturation Score: z
[*] Crawl Efficiency:z  Avg New Terms per Page: z  Information Saturation: zQ
--------------------------------------------------------------------------------z*[bold cyan]DETAILED STATISTICS[/bold cyan]zDETAILED STATISTICSzP--------------------------------------------------------------------------------z
[+] Top 20 Terms by Frequency:c                     U S   $ r   r%   r!  s    r'   r(   -AdaptiveCrawler.print_stats.<locals>.<lambda>  s    VWXYVZr&   r$  r   r   z  2dz. [yellow]'z'[/yellow]: z occurrences in z docsz. 'z': z
[+] URLs Crawled (z):z. [cyan]z[/cyan]z     -> Added [green]z[/green] new termsz. z     -> Added z
 new termsz%
[+] Document Frequency Distribution:z  Terms in z docs: z termsz 
[+] Semantic Coverage Analysis:z  Embedding Model: z  Query Variations: z  Knowledge Embeddings: z  Knowledge Embeddings: Nonez  Semantic Gaps: z  Coverage Achievement: z
[+] Query Space (samples):r   z
. [yellow]z	[/yellow]))r^   rM  rich.consoler|  
rich.tabler}  ImportErrorr   
add_columnru  add_rowrk   rZ   r   r   ra  r    rx  r  r   r   r0   r  r   r*   r   r   r   r   r,   r1   r  rG   r  r2   r@  r8   r5   r3   r  r7   )rL   rz  r|  r}  consoleuse_richtablestatsrr  total_wordsrs  r  r  r	  r
  statusstatus_coloredavg_new_terms	top_termsr  freqrh   rB  	df_countscounteqs   `                         r'   print_statsAdaptiveCrawler.print_stats"  s9    zz01	,(iGH H"A$**BRBRASST UVEXVTBWI6 ''EMM/3uyy!/L+MNMM.#eii.J*KLMM-UYY}a-H)IJMM*uyy9OQR/STU.VV\,]^MM/3uyy!/L+MNMM"b! $--):;;luyyq/I#.NP0TZZ5G5G5K5KL^`a5bcf4gi3

8J8J8N8NOdfg8hil7mo0TZZ5G5G5K5KLcef5ghk4lnb"%.TXTfTf0Pl{| luyyq/I#.NPjUYYz1-Ec,JLm		-0KC/PRluyyq/I#.NPb"%.HZHZ0D`opMM%  - 89I9I8J!LM&M +,%c$***A*A&B%CDE%c$***B*B&C%DEF'

(B(B'CDE $' '"jj77' $  djj99@@BCKtzz::;L-/%&:1%=[IJOK?34$\!$456Q/0H/MNO $--):;;9;01C1C1G1GHZ\]1^_b0cde7

8J8J8N8NOdfg8hil7mno<TZZ=O=O=S=STnpq=rsv<wxy 13%%(>ts>SSn&op 6ts6K<XY(>ts>SSn&op 6ts6KK[\]*4::+=+=+A+ABRTU+VWZ*[\],TZZ-?-?-C-CD[]^-_`c,def -/"mm55djj6F6F6L6L6NO'D4488qAB88<<T1EBAv##MMCv\"QtzzGaGaFbbpqstxt~t~  uO  uO  rO  PS  qT  T_  `b  _c  co  +p  q!Cv\"Qtzz?Y?Y>ZZabdeieoeoeeb  AD  bE  EH  IK  HL  LX  #Y  Z##MMCv5P*QR!Cv-=">? ( 13#'#5#56>B>P>P%:VgNMM$:4??3:OqQ_P`"ab24??32GqQR*4::+=+=+A+A*a+PQT*UVW-djj.@.@.D.D]TU.VWZ-[\],TZZ-?-?-C-CLRS-TUX,YZ[ zz++ #DJJ$@$@ AC

HdHdDe e/12=2EFG24::3E3E3I3I,XY3Z[^2_`am$MM"NO/0f 89"4::#>#>#D#D#FNdhijmkmn	'0A'>OA|d88<<T1EB1R&D6dVScdfcggl&mn1R&D6TF:J2$eTU (? ,S1H1H-I,J"MN'

(>(>BFAsEF#djjNjNjJkEk

 < <QqS AqrI1#XcU'&BC(=i[HZ&[\1#Ruo.ykDE C >?	**99@@BB$-MM"a$81$<IbM C !!12B%bMEKt75'@A 3
 ::---=>/

0J0J/KLM0TZZ5P5P1Q0RSTzz//; 89Q9Q9W9W8XYZ <>-c$**2J2J.K-LMN4T__S4IJK zz22<=%.tzz/J/J2A/NPQ%REAr' '1#Zt9.M N %1#Rtn 5	 &S - s  	H	s   w6 6xxc                     [        US5      (       a`  UR                  (       aO  [        UR                  S5      (       a  UR                  R                  =(       d    S$ [        UR                  5      $ g)z,Helper to safely extract content from resultrc   rd   r   )rj   rc   rd   rk   )rL   rO  s     r'   r  (AdaptiveCrawler._get_content_from_result  sM    6:&&6??v77339r9v''r&   filepathformatc           	      6   U R                   (       a  U R                   R                  (       d  [        S5        g[        U5      nUR                  R                  SSS9  US:X  a  [        USSS9 nU R                   R                   H;  nU R                  U5      nUR                  [        R                  " US	S
9S-   5        M=     SSS5        [        S[        U R                   R                  5       SU 35        g[        SU 35      e! , (       d  f       NK= f)zExport the knowledge base to a file

Args:
    filepath: Path to save the file
    format: Export format - currently supports 'jsonl'
zNo knowledge base to export.NTr;   jsonlr>   utf-8encodingF)ensure_ascii
z	Exported z documents to zUnsupported export format: )r^   r   rM  r   rA   rB   rI   _crawl_result_to_export_dictwriterJ   dumpsr  r>  )rL   r  r  rR   rO  result_dicts         r'   export_knowledge_base%AdaptiveCrawler.export_knowledge_base  s     zz!:!:01>dT:Whg6!"jj77F"&"C"CF"KKGGDJJ{G$NO	 8 7 Ic$**";";<=^H:VW:6(CDD 76s   /AD


Dc                 B   [        USS5      [        USS5      [        USS5      U R                  (       a  U R                  R                  OSS.n[        US5      (       a_  UR                  (       aN  [        UR                  S	5      (       a  UR                  R
                  US
'   O[        UR                  5      US
'   OSUS
'   [        US5      (       a  UR                  US'   [        US5      (       a  UR                  US'   U R                  (       a  US   U R                  R                  ;   a+  U R                  R                  R                  US   5      S-   OSU R                  R                  R                  SS5      U R                  R                  S.US'   U$ )z.Convert CrawlResult to a dictionary for exportrh   r   	timestampNrC  T)rh   r  rC  r   rc   rd   ri   rf   re   r   r   r   )r2   confidence_at_crawlr0   crawl_metadata)r  r^   r   rj   rc   rd   rk   rf   re   r2   indexr    rZ   r0   )rL   rO  export_dicts      r'   r  ,AdaptiveCrawler._crawl_result_to_export_dict  sj    65"- d;vy$7)-TZZ%%	
 6:&&6??v77)/)E)EI&),V__)=I&%'K	" 6:&&&,ooK
# 67###)<<K  ::WbchWimqmwmw  nD  nD  XDtzz55;;K<NORSS  JK'+zz'9'9'='=lA'N#'::#=#=-K() r&   c                   #    [        U5      nUR                  5       (       d  [        SU 35      eUS:X  a  / n[        USSS9 nU HR  nUR	                  5       (       d  M  [
        R                  " U5      nU R                  U5      nUR                  U5        MT     SSS5        U R                  (       d  [        5       U l	        U R                  R                  R                  U5        U R                  R                  U R                  U5      I Sh  vN   [        S[!        U5       SU 35        g[#        S	U 35      e! , (       d  f       N= f N>7f)
zImport a knowledge base from a file

Args:
    filepath: Path to the file to import
    format: Import format - currently supports 'jsonl'
zFile not found: r  rW   r  r  Nz	Imported z documents from zUnsupported import format: )r   existsFileNotFoundErrorrI   r  rJ   r  _import_dict_to_crawl_resultr   r^   r   r   rI  r   r   rM  r  r>  )rL   r  r  imported_resultsrR   linedatamock_results           r'   import_knowledge_base%AdaptiveCrawler.import_knowledge_base5  s     >  #&6xj$ABBW!hg6!Dzz||#zz$/&*&G&G&M(//<  7 ::'\
 JJ%%,,-=> --,,TZZ9IJJJIc"2344DXJOP:6(CDD) 76  Ks1   AEE!<EA6EE-E
EEr  c                 B   ^  " S S5      m " U4S jS5      nU" U5      $ )z0Convert imported dict back to a mock CrawlResultc                       \ rS rSrS rSrg)BAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockMarkdowniZ  c                     Xl         g r"   rs   rt   s     r'   ru   KAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockMarkdown.__init__[  rw   r&   rs   Nrx   r%   r&   r'   r~   r  Z  r   r&   r~   c                   "   > \ rS rSrU 4S jrSrg)EAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockCrawlResulti^  c                 $  > UR                  SS5      U l        T" UR                  SS5      5      U l        UR                  S0 5      U l        UR                  S0 5      U l        UR                  SS5      U l        UR                  S5      U l        g )	Nrh   r   ri   re   rf   rC  Tr  )rZ   rh   rc   re   rf   rC  r  )rL   r  r~   s     r'   ru   NAdaptiveCrawler._import_dict_to_crawl_result.<locals>.MockCrawlResult.__init___  sr    88E2. ,TXXi-D E!XXgr2
 $R 8#xx	48!%+!6r&   )re   rc   rf   rC  r  rh   Nrx   r   s   r'   r   r  ^  s    7 7r&   r   r%   )rL   r  r   r~   s      @r'   r  ,AdaptiveCrawler._import_dict_to_crawl_resultX  s$    	, 	,	7 	7 t$$r&   top_kc                    U R                   (       a  U R                   R                  (       d  / $ / n[        U R                   R                  R	                  5       R                  5       5      n[        U R                   R                  5       H  u  pEUR                  R                  =(       d    SR	                  5       n[        UR                  5       5      n[        X7-  5      nU(       a  U[        U5      -  OSn	UR                  UR                  U	UR                  R                  US.5        M     UR                  S SS9  USU $ )z'Get most relevant content for the queryr   r   )rh   r/  ri   r  c                     U S   $ )Nr/  r%   r!  s    r'   r(   6AdaptiveCrawler.get_relevant_content.<locals>.<lambda>  s    qzr&   Tr$  N)r^   r   r.   r   r   rY  r  rc   rd   r  r   rh   r*  )
rL   r  scored_docsr  r  rO  ri   content_termsr  r/  s
             r'   get_relevant_content$AdaptiveCrawler.get_relevant_contenti  s   zz!:!:I $****00288:;"4::#<#<=IA339r@@BG0M +56G2=Gc+..3Ezz!??77	   >  	14@6E""r&   )r:  r   r7  r^   r   r4  r"   )F)r  )r   )'ry   rz   r{   r|   r   r   r   r   r   ru   rk   r9  r   rS  r   rH  r   r	   r   r   rJ  r   r   r   r
   ru  r   rx  r  r  r   r   r  r  r  r  r$   r  r}   r%   r&   r'   r6  r6    s   F 7;4859-"?3-!.1- $M2-(Cc Cm C$ =AN?*-N?&)N? -5SMN? FPN?`S  +AV @DtU{9K4L UX ]abm]n 0 E   
S#X 
 
, Gt G GI!D I!T I!V# EeCI.> E EZ^ E6#d38n #J!EE#t)4D !Ec !E`d !EF%c3h %"## #d4S>6J # #r&   r6  )+r   abcr   r   typingr   r   r   r   r	   r
   r   dataclassesr   r   rg  pickleosrJ   r   collectionsr   r   rW  pathlibr   crawl4ai.async_webcrawlerr   crawl4ai.async_configsr   r   r   crawl4ai.modelsr   r   numpyr[   r   r   r   r   ra  r6  r%   r&   r'   <module>r     s    $ ? ? ? (   	   , 	  5 Q Q - 
{
 {
 {
| x/ x/ x/vC 0w/- w/t	b
Z b
ZJw	# w	#r&   