MiSSSKrSSKJr SSKrSSKJrJrJrJrJ r SSK r SSK J r SSK r SSK r SSKrSSKJr SSKJr SSKJr SSKJr SS KJr S S KJr SSKrSSKrSSKrSSKrS \lS \S\ 4Sjr!"SS5r"g)N)Path)DictListTupleOptionalAny)tqdm) BM25Okapi) word_tokenize) stopwords)WordNetLemmatizer)batch_completion) AsyncLoggerF file_pathreturnc^[R"5nURS5m[U4SjS5HnUR U5 M SSS5 UR 5$!,(df  UR 5$=f)z/Compute MD5 hash for the file's entire content.rbc&>TRS5$)Ni)read)fsp/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/crawl4ai/legacy/llmtxt.py$_compute_file_hash..s !&&,rN)hashlibmd5openiterupdate hexdigest)rhash_md5chunkrs @r_compute_file_hashr$sj{{}H  .4E OOE "5           s )A)) Bc b\rSrSrS$S\S\\S\S\SS4 SjjrS \ \SS4S jr S \ S\ \ \\ 44S jrS \S\4SjrS \S\SS4SjrS\ S\ \ 4SjrS%S\ 4SjjrS%S&SjjrS'S\ S\ SS4SjjrS(S\ \ S\ S\ 4SjjrS)S\ S\S\ 4SjjrS\ \S\S \ \ S\4S!jrS&S"jrS#rg)*AsyncLLMTextManager!Ndocs_dirloggermax_concurrent_calls batch_sizercXlX lX0lX@lSUl0Ul/UlURS- Ulg)Nzbm25_index.pkl)r(r)r*r+ bm25_index document_maptokenized_factsbm25_index_file)selfr(r)r*r+s r__init__AsyncLLMTextManager.__init__"sB!  $8!$,.*,#}}/??r doc_batchc V# /nUH7n[USSS9nURUR55 SSS5 M9 SnUVs/sHnU(dM S US U3S ./PM nn[S USS 9n [X5GH&up[R"SU RSRR[R5n U (d URRSU35 Mu[R "SSU R#S55R%5n U (aWUR'S5n [U SSS9nUR)U 5 SSS5 URR+SU 35 GMURRSU35 GM) g!,(df  GM=f![aGnURR SUS[ U535 URS5 SnAGMSnAff=fs snf!,(df  N=f![a6nURR SUS[ U535 SnAGMSnAff=f![a1nURR S[ U535 SnAgSnAff=f7f)z(Process a batch of documents in parallelrutf-8encodingNError reading : aGiven a documentation file, generate a list of atomic facts where each fact: 1. Represents a single piece of knowledge 2. Contains variations in terminology for the same concept 3. References relevant code patterns if they exist 4. Is written in a way that would match natural language queries Each fact should follow this format: : | | Example Facts: browser_config: Configure headless mode and browser type for AsyncWebCrawler | headless, browser_type, chromium, firefox | BrowserConfig(browser_type="chromium", headless=True) redis_connection: Redis client connection requires host and port configuration | redis setup, redis client, connection params | Redis(host='localhost', port=6379, db=0) pandas_filtering: Filter DataFrame rows using boolean conditions | dataframe filter, query, boolean indexing | df[df['column'] > 5] Wrap your response in ... tags. userz* Generate index for this documentation: )rolecontentz"anthropic/claude-3-5-sonnet-latest)modelmessages logger_fnz(.*?)rz(No ... content found for z\n\s*\n r.q.mdwzCreated index file: z'No index content found in response for zError processing response for zError in batch completion: )rappendr Exceptionr)errorstrrzipresearchchoicesmessager?DOTALLwarningsubgroupstrip with_suffixwriteinfo)r1r4contentsrrepromptr? messages_list responsesresponseindex_content_match index_content index_files r_process_document_batch+AsyncLLMTextManager._process_document_batch2sz"I $)S7;qOOAFFH-<;#2$  $ ""()WX_W`a  $   ( F(:&I(+9'@#*,))/ ((+33;; +' / ++FykR!$&FF"D*=*C*CA*F%eg"%%.%:%:7%C !*cGDGGM2E ((+? |)LM ++Ei[Q-(AW<; $ !!N9+RAx"HI## $,  JED!KK%%8 2c!fXN  F KK   ;CF8D E E FsJ) F> F+F> J) HH&J))I+A,H(2I+4AH( H&H(I+H(%I+*J)+ F; 5F>8J);F>> H;H J) HJ) H% !H(( I(2*I#I+#I((I++ J&5'J!J)!J&&J)linecSU;agURS5Vs/sHo"R5PM nn[U5S:waSS[U534$USnSU;agg s snf) N|)FzMissing separator '|'FzExpected 3 parts, got r:)Fz!Missing ':' in concept definition)TN)splitrSlen)r1rbpparts concept_parts r_validate_fact_line'AsyncLLMTextManager._validate_fact_linesj d?1$(JJsO4OqO4 u:?23u:,?? ?Qx l "=5sA" fact_filec >URS5n[U5nUR5(ab[US5n[R "U5nSSS5 WR S5U:XaU$URRSUS35 0US .$!,(df  NI=f![Ra" URRSUS35 NK[a4nURRS US [U535 SnANSnAff=f) z} Load token cache from .q.tokens if present and matching file hash. Otherwise return a new structure with updated file-hash. .q.tokensr6N content_hashzHash changed for z, reindex needed.zCorrupt token cache for z , rebuilding.zError reading cache for r;factsrq) rTr$existsrjsonloadgetr)rVJSONDecodeErrorrPrGrI)r1rn cache_file current_hashrcacherXs r_load_or_create_token_cache/AsyncLLMTextManager._load_or_create_token_caches **;7 ))4      V*c*a IIaLE+99^, < L   #4YK?P!QR\::+*'' Y ##&>yk$WX V ##&>ykCPQF8$TUU Vs: B*BB*5B* B'#B**3D D(*DDr{cURS5n[U5US'[US5n[R"X$5 SSS5 g!,(df  g=f)NrprqrE)rTr$rrudump)r1rnr{ryrs r_save_token_cache%AsyncLLMTextManager._save_token_cachesE**;7 29 =n *c "a IIe # " "s A  Atextc BSU;a/URS5Vs/sHo"R5PM snOU/n[R"SSUS5US'[ 5n[ [ R"S551Sk- n/nUHnSU;a.SU;a([R"S U5nURU5 [UR55n URU V s/sHn X;dM URU 5PM sn 5 M U$s snfs sn f) Nrdz^(.*?):z\1renglish>howwhywhatwhenwherewhich()z.[\w_]+(?=\()|[\w_]+(?==[\'"]{1}[\w_]+[\'"]{1})) rgrSrKrQr setr wordsfindallextendr lower lemmatize) r1rxrj lemmatizer stop_wordstokenspart code_tokensrtokens rpreprocess_text#AsyncLLMTextManager.preprocess_texts8;t DJJsO4OqO4$66*eU1X6a&( 348  Dd{sd{ jjEt  k*!$**,/E MM"'!&.0J((/!&   ?50sD( D 5D cZU(d[RRUR5(aeURR S5 [ URS5n[R"U5nSSS5 WSUl USUl gg!,(df  N$=f)zG Load existing BM25 index from disk, if present and clear_cache=False. z&Loading existing BM25 index from disk.rNr/r-TF) ospathrtr0r)rVrpicklervr/r-)r1 clear_cacherdatas rmaybe_load_bm25_index)AsyncLLMTextManager.maybe_load_bm25_indexsrww~~d.B.BCC KK  E Fd**D1Q{{1~2#'(9#:D "<0DO 21s 'B B*c  U(aTURRS5 URR5(aURR 5 [ R "5nURRS5 [R"UR5Vs/sH)o3RS5(dMURU- PM+ nn/n/n/n/nUHn U RS5n U(dU R5(dURU 5 MCURU 5n [U S5S:XdU RS5[!U 5:waURU 5 MU SR#5H8upURU 5 URU S5 XR$U 'M: M U(GdU(dUR'S S 9(aURRS 5 g URRS 5 U(aURRS[U5S35 [)U5UlXPl[/URS5n[0R2"UR*UR,S.U5 S S S 5 g URR5S5 g URR[U5S35 /n/n[7[U5SS9nUGHn0[!U5S.n[/USSS9nUR95R;5nUR=S5Vs/sH*nUR;5(dMUR;5PM, nnS S S 5 WHn UR?U 5unnU(dURUU U45 M4URAU 5nU[BRB"5S.USU 'URU 5 URU5 UUR$U 'M UREUU5 URG5RHS- S- nURRKSURLSUSS 35 URUS"5 GM S S S 5 U(a\URR5S#[U5S$35 UH.unn nURR5USUS%U S S&S'35 M0 X^-nXo-nURRS[U5S(35 [)U5UlUUl[/URS5n[0R2"UR*UR,S.U5 S S S 5 URG5RHS- S- nURRS)USS 35 g s snf!,(df  g =fs snf!,(df  GNy=f![Na5nURRQS!US[SU535 S nAGNS nAff=f!,(df  GN=f!,(df  N=f)*z Checks for new or modified .q.md files by comparing file-hash. If none need reindexing and clear_cache is False, loads existing index if available. Otherwise, reindexes only changed/new files and merges or creates a new index. z0Clearing cache and rebuilding full search index.z/Checking which .q.md files need (re)indexing...rDrprsrrqrFrz"a'599,#B',(r").g(<(<(>$D"))$/#**:h+?@.0%%d+)?#,{))e)<   R   O"KK$$3C4G3HW'0&@DO+9(d22D9Q .2oo373G3G :KK''(UV CL>)QRS  G +C D (*# UHnTRU5v M g7fN)r).0rrs r ;AsyncLLMTextManager.generate_index_files..s,XDWqQZZ]]DWs!)rD.xs.mdrDz4All index files exist. Use force=True to regenerate.rzProcessing batch r/Nz:Index generation complete, building/updating search index.r)r)rVrrr(ranyrreplacertrangerhr+r`r)r1rrrmd_filesibatchs ` rgenerate_index_files(AsyncLLMTextManager.generate_index_files}s MNZZ . .zz%  ),,XDW,X)X DMMA .  $"!A ug(FFNNP!   KK  S T1c(mT__= Q%89   '??(:Q(>'?q#h-QUQ`Q`B`deAe@fg225999 > UV ,<=5 :s=?GG G<G GAG G&B5GG>Gsectionsmodec ^[R"[URS- 55[R"[URS- 55-nUVs1sHUn[U5RR S5(aM.[U5RR S5SiMW nnU(a/UV^s1sH!m[U4SjU55(dMTiM# nn/n[USS9HmUS :XaWURTS 3- nURTS 3- n UR[UR5(aUOU 55 M`UR[URTS 3- 55 M /n UHWn [U S S S9n[U 5Rn U RSSU SSSUR535 SSS5 MY U (aSR!U 5$S$s snfs snf!,(df  M=f![a5n URRSU S[U 535 Sn A MSn A ff=f)Nz [0-9]*.mdz [0-9]*.xs.mdrD.rc3d># UH%oR5TR5;v M' g7fr)r)rsectiondocs rr/AsyncLLMTextManager.generate..s NX'}}#))+5Xs-0cURS5SR5(a[URS5S5$S$)N_ri?B)rgisdigitintrs rr.AsyncLLMTextManager.generate..s8!''#,q/2I2I2K2K#aggcl1o.WQWWr)key condensedrrr6r7r8####################z # rCz r:r; --- r<)globrIr(rrrrgrsortedrFrtrrrGr)rHjoin)r1rr all_filesr base_docsrfilesxs_file regular_filer?rfnamerXs ` rgenerateAsyncLLMTextManager.generatesIIc$--+"=>?$))  . /C   7<<((1 'DGLL  s #A &   %$CNXNN$  W C{"--SE.8#}}#c{:  SGNN,<,<,OP SC5!<=> D E$g6! JOOENNfXT%6($qvvxj#QR76/6}!!'*=2=K .76 E !!N4&3q6("CDD EsO-H+H H +H  H#!AH!H# H H# H## I"-*II"querytop_kcUR(dgURU5nURRU5n[R"U5n[R "U5nUSU--nUR UUUS9n[UR5SSS9SUn /n U Hup[U 5RSS5n [RRURU - 5(dMS[URU - S S S 9nU R!S 5S nSSU3SSUR#5/nU R%SR'U55 SSS5 M SR'U 5$!,(df  M=f)Nz;No search index available. Call build_search_index() first.g?) doc_scoresscore_threshold query_tokenscBUSSS-USSS--USS-$)Nrcode_match_scoreg@ match_countg? total_scorers rr,AsyncLLMTextManager.search..s;!'(3.A$}%+,A$}%&rT)rreverserDrr6r7r8rrz# r<rCr)r-r get_scoresnpmeanstd_aggregate_search_scoresrrrIrrrrtr(rrgrrFr)r1rrrr mean_score std_scorer file_data ranked_filesresultsrrmain_docronly_file_namer?s rrLAsyncLLMTextManager.searchsmP++E2 __// = WWZ( FF:& $y(8911!+%2  OO    5 #GD4y((%8Hww~~dmmh677$--(2C'Ja%-^^C%8%"affhWGNN499W#56KJ$!!'** KJs A E:: F rrrc"0n[U5HupVXb::aM URUnURUnX;a SSS/S.XH'SU;aURS5OU/n Sn [ U 5S:XaPU SR 5n UR U 5n [ [U5[U 5-5[ U5- n XHS==U- ss'XHS==S- ss'[XHS U 5XHS 'XHS RU5 M U$) Nr)rrr matched_factsrdrerrrrr) enumerater/r.rgrhrSrrmaxrF) r1rrrridxscorefactr componentsrcode_refrs rr,AsyncLLMTextManager._aggregate_search_scoressA #J/JC''',D))$/I)#$#$()%' ( $-04KCdVJ :!#%a=..0"228< #&s<'83{;K'K#Ls P$    /5 8 /   /1 4 /7:$%78:J8I !3 4   1 8 8 >=0@rc"URSS9 g)z&Convenience method for a full rebuild.TrN)r)r1s r refresh_index!AsyncLLMTextManager.refresh_index s D1r)r+r-r0r(r.r)r*r/)Nre)F)rN)FF)extended)r()__name__ __module__ __qualname____firstlineno__rrrrr2rr`rIrboolrlrr|rrrrrrrLfloatrr&__static_attributes__r rrr&r&!sx)-$% @@%@" @  @  @ PFtDzPFdPFd  dHSM6I0J ;T;d;2 4    C DI D $ ZYzLQ&>$(&>DH&> &>P,>c,>#,>s,>\$+C$+$+C$+L%u+%8=%MQRUY% %N2rr&)#rpathlibrrKtypingrrrrrrur rrnumpyr rank_bm25r nltk.tokenizer nltk.corpusr nltk.stemr litellmr async_loggerrrrr set_verboserIr$r&r rrr;sh  33  '!'$%   $ 3 A2A2r