
    j(                        S SK r S SKrS SKJr  S SKJr  S SKJrJrJ	r	  S SK
Jr  S SKJr  S SKrS SKrS SKJr  S SKrS rS	 rSS
 jrS rS rS rS rS rS rS rSS jr S r\S:X  a  Sr S\  3r!Sr"Sr#Sr$\" \ \!\#\$5        gg)    N)OllamaEmbeddings)FAISS)PyPDFLoader
TextLoaderDocx2txtLoader)RecursiveCharacterTextSplitter)Presentation)SemanticChunkerc                 `    U R                  SS5      n[        R                  " SSSSU /5        U$ )N.ppt.pptxlibreofficez
--headlessz--convert-topptx)replace
subprocessrun)	file_pathpptx_file_paths     C/var/www/eduai.edurigo.com/question_generate/production/training.pyconvert_ppt_to_pptxr      s1    &&vw7NNNM<ST    c                    [         R                  R                  U 5      S   R                  5       nUS:X  a  [	        U 5      nUR                  5       nGOUS:X  a  [        U SS9nUR                  5       nOUS;   a)  [        R                  " U 5      R                  S5      nU/nOUS:X  at  [        U 5      n/ nUR                   HD  nUR                   H1  n[        US5      (       d  M  UR                  UR                  5        M3     MF     S	R!                  U5      /nOBUS
:X  a.  [#        U 5      n[%        U5      n[         R&                  " U5        U$ [)        SU 35      eU(       d  [+        S5        / $ [-        SSS9n	/ n
U Hi  n[/        U[0        5      (       a  UnO,[        US5      (       a  UR2                  nO[+        SU5        MG  U	R5                  U5      nU
R7                  U5        Mk     U
$ )N   z.pdf.txtutf-8encoding)z.docxz.docr   text
r   zUnsupported file type: zNo documents loaded from file.i  2   )
chunk_sizechunk_overlappage_contentUnsupported document format:)ospathsplitextlowerr   loadr   textractprocessdecoder	   slidesshapeshasattrappendr   joinr   load_and_split_documentremove
ValueErrorprintr   
isinstancestrr#   
split_textextend)r   file_extensionloaderdocsr   prsslideshaper   text_splitterfinal_documentsdocchunkss                r   r2   r2      s   WW%%i0399;NY'{{}	6	!I8{{}	,	,	*11':v	7	"9%ZZE5&))KK

+ &   		$ 	6	!,Y7&~6
		.!2>2BCDD./	2cQSTMOc3DS.))##D0#6))$/v&  r   c                    [        US9n/ nU  He  n[        U[        5      (       a  UR                  U5        M+  [	        US5      (       a  UR                  UR
                  5        MY  [        SU5        Mg     [        R                  " X2S9nU$ )Nmodelr#   r$   )texts	embedding)	r   r6   r7   r0   r/   r#   r5   r   
from_texts)	documentsrF   
embeddingsrG   rB   vectorss         r   create_embeddingsrM   C   su    !.JEc3LLS.))LL))*0#6  UAGNr   c                 &    [        S U  5       5      $ )Nc              3   T   #    U  H  n[        UR                  5       5      v   M      g 7f)Nlensplit.0rB   s     r   	<genexpr>count_tokens.<locals>.<genexpr>S   s     59Cs399;9s   &(sum)rJ   s    r   count_tokensrY   R   s    59555r   c                 8    [        U 5      nUR                  5       $ )zLoad PDF and return documents.)r   r)   )r   r;   s     r   load_pdfr[   [   s     #F;;=r   c                 l    SnU  H+  nUR                   R                  5       nU[        U5      -  nM-     U$ )z+Counts total tokens in the given documents.r   )r#   rR   rQ   )rJ   total_tokensdocumenttokenss       r   count_tokens_in_documentsr`   a   s;    L&&,,.F#  r   c                 &    [        S U  5       5      $ )z1Count the total number of words in the documents.c              3   Z   #    U  H!  n[        US    R                  5       5      v   M#     g7f)r#   NrP   rS   s     r   rU   $count_total_words.<locals>.<genexpr>l   s&     @4Cs3~&,,.//4s   )+rW   )r<   s    r   count_total_wordsrd   j   s    @4@@@r   c                     [        USS9nUR                  U  Vs/ s H  o3R                  PM     sn5      n[        S5        U$ s  snf )z@Splits the text into semantic chunks using the given embeddings.
percentile)breakpoint_threshold_typez%Documents split into semantic chunks.)r
   create_documentsr#   r5   )r<   rK   r@   rB   rJ   s        r    split_text_with_semantic_chunkerri   n   sK    #lM ..D/QDS0@0@D/QRI	
12 0Rs   Ac                    [         R                  R                  U5      (       d  [         R                  " U5        [	        U 5       Hj  u  p#SUS-    S3n[         R                  R                  X5      n[        USSS9 nUR                  UR                  5        SSS5        [        SU 35        Ml     g! , (       d  f       N!= f)	zBSaves each document in the documents list as a separate .txt file.document_part_r   r   wr   r   NzSaved: )
r%   r&   existsmakedirs	enumerater1   openwriter#   r5   )rJ   
output_dirir^   	file_namer   files          r   save_documents_to_txtrv   x   s    77>>*%%
J +$QqSE.	GGLL7	)S73tJJx,,- 4 		{#$ , 43s   >B55
C	c           	         [        U5      n[        U5      n[        R                  R                  SX5      n[        R                  " USS9  [        SS9n[        U SS9 H  u  pV[        R                  R                  U5      n[        R                  " USS9  [        R                  " U/US9n[        R                  R                  US	U 35      n	UR                  U	5        [        S
U SU SU 35        M     g )Nmy_embeddingsT)exist_oknomic-embed-textrE   r   )start)rH   faiss_indexz(Saved FAISS embedding for document part z as faiss_indexz in )r7   r%   r&   r1   rn   r   ro   r   from_documents
save_localr5   )
split_documents	client_idreference_idembedding_folder_baserK   idxrB   embedding_foldertemp_dbembedding_file_paths
             r   create_and_save_embeddingsr      s    II|$LGGLL)Q KK%5 "(:;J oQ777<<(=> 	$t4 &&u
C !ggll+;{3%=PQ./8_SEQUVfUghi 8r   c                    [        SS9nS nU SU  SU 3n[        R                  " U5       Vs/ s H@  nUR                  S5      (       d  M  U[	        S5      S  R                  5       (       d  M>  UPMB     nn[        US S9nU H  n[        R                  R                  XV5      n	[        SU	 35        [        R                  " XSS	9n
U
R                  R                  5        Vs/ s H(  nU
R                  R                  U5      R                   PM*     nnUc  U
nM  UR#                  U5        M     Ub  UR%                  U S
35        [        U5        U HG  n[        R                  R                  XV5      n	 [&        R(                  " U	5        [        SU	 35        MI     U$ s  snf s  snf ! [*         a    [        SU	 35         Mr  [,         a  n[        SU	 SU 35         S nAM  S nAff = f)Nrz   rE   /r|   c                 8    [        U R                  SS5      5      $ )Nr|    )intr   )xs    r   <lambda>!merge_all_faiss.<locals>.<lambda>   s    QYY}VX=Y9Zr   )keyzLoading FAISS index from: T)allow_dangerous_deserializationz/merged_faisszDeleted FAISS index folder: zFolder not found: zError deleting z: )r   r%   listdir
startswithrQ   isdigitsortedr&   r1   r5   r   
load_localindex_to_docstore_idvaluesdocstoresearchr#   	add_textsr~   shutilrmtreeFileNotFoundErrorOSError)r   r   	base_pathrK   merged_faissfolder_pathfolderfaiss_folderssorted_folders
faiss_pathcurrent_faissdoc_idcurrent_textses                 r   merge_all_faissr      s   &89JL Kq1\N;K  ZZ446]+ 	06s=7I7J0K0S0S0U 	4   M/Z[N !WW\\+6
*:,78((aef (5'I'I'P'P'RT'RV '//66v>KK'R 	 T (L ""=1 !  ;-} =>	, !WW\\+6
	7MM*%0=> ! QT* ! 	5&zl34 	7OJ<r!566	7s5   F#F#/F#%/F(9$F--G-	G-G((G-c                     [         R                   " 5       n[        SU 35        [        U 5      n[        U5      n[	        SS9n[        XW5      n[        X5        [        XU5        SU SU 3n	 [        X#5      n
g )NzStart Time: rz   rE   zmy_embeddings/r   )	timer5   r[   r`   r   ri   rv   r   r   )r   rr   r   r   
start_timer<   Total_countrK   r   embedding_dirmerge_embeddingss              r   mainr      s    J	L
%& ID+D1K ");<J 7tHO /6 <H %YKq?MT
 'y>r   __main__z	cyber.pdfztemp/output_embeddingsi  iy  )rz   )rx   )%r   r%   langchain_ollamar    langchain_community.vectorstoresr   $langchain_community.document_loadersr   r   r   langchain_text_splittersr   r   r	   r*   r   $langchain_experimental.text_splitterr
   r   r   r2   rM   rY   r[   r`   rd   ri   rv   r   r   r   __name__r   rr   r   r   r    r   r   <module>r      s     	 - 2 X X C    @ 
.`6A%j>1hH.!?F zI$J/ILJ	,7 r   