
    j(                        S SK r S SKrS SKJr  S SKJr  S SKJrJrJ	r	  S SK
Jr  S SKJr  S SKrS SKrS SKJr  S SKrS rS	 rSS
 jrS rS rS rS rS rS rS rSS jr S r\S:X  a  Sr S\  3r!Sr"Sr#Sr$\" \ \!\#\$5        gg)    N)OllamaEmbeddings)FAISS)PyPDFLoader
TextLoaderDocx2txtLoader)RecursiveCharacterTextSplitter)Presentation)SemanticChunkerc                 `    U R                  SS5      n[        R                  " SSSSU /5        U$ )N.ppt.pptxlibreofficez
--headlessz--convert-topptx)replace
subprocessrun)	file_pathpptx_file_paths     @/var/www/eduai.edurigo.com/question_generate/testing/training.pyconvert_ppt_to_pptxr      s1    &&vw7NNNM<ST    c                    [         R                  R                  U 5      S   R                  5       nUS:X  a  [	        U 5      nUR                  5       nGOUS:X  a  [        U 5      nUR                  5       nOUS;   a)  [        R                  " U 5      R                  S5      nU/nOUS:X  at  [        U 5      n/ nUR                   HD  nUR                   H1  n[        US5      (       d  M  UR                  UR                  5        M3     MF     SR!                  U5      /nOBUS	:X  a.  [#        U 5      n[%        U5      n[         R&                  " U5        U$ [)        S
U 35      eU(       d  [+        S5        / $ [-        SSS9n	/ n
U Hi  n[/        U[0        5      (       a  UnO,[        US5      (       a  UR2                  nO[+        SU5        MG  U	R5                  U5      nU
R7                  U5        Mk     U
$ )N   z.pdf.txt)z.docxz.docutf-8r   text
r   zUnsupported file type: zNo documents loaded from file.i  2   )
chunk_sizechunk_overlappage_contentUnsupported document format:)ospathsplitextlowerr   loadr   textractprocessdecoder	   slidesshapeshasattrappendr   joinr   load_and_split_documentremove
ValueErrorprintr   
isinstancestrr!   
split_textextend)r   file_extensionloaderdocsr   prsslideshaper   text_splitterfinal_documentsdocchunkss                r   r0   r0      s   WW%%i0399;NY'{{}	6	!I&{{}	,	,	*11':v	7	"9%ZZE5&))KK

+ &   		$ 	6	!,Y7&~6
		.!2>2BCDD./	2cQSTMOc3DS.))##D0#6))$/v&  r   c                    [        US9n/ nU  He  n[        U[        5      (       a  UR                  U5        M+  [	        US5      (       a  UR                  UR
                  5        MY  [        SU5        Mg     [        R                  " X2S9nU$ )Nmodelr!   r"   )texts	embedding)	r   r4   r5   r.   r-   r!   r3   r   
from_texts)	documentsrD   
embeddingsrE   r@   vectorss         r   create_embeddingsrK   C   su    !.JEc3LLS.))LL))*0#6  UAGNr   c                 &    [        S U  5       5      $ )Nc              3   T   #    U  H  n[        UR                  5       5      v   M      g 7f)Nlensplit.0r@   s     r   	<genexpr>count_tokens.<locals>.<genexpr>S   s     59Cs399;9s   &(sum)rH   s    r   count_tokensrW   R   s    59555r   c                 8    [        U 5      nUR                  5       $ )zLoad PDF and return documents.)r   r'   )r   r9   s     r   load_pdfrY   [   s     #F;;=r   c                 l    SnU  H+  nUR                   R                  5       nU[        U5      -  nM-     U$ )z+Counts total tokens in the given documents.r   )r!   rP   rO   )rH   total_tokensdocumenttokenss       r   count_tokens_in_documentsr^   a   s;    L&&,,.F#  r   c                 &    [        S U  5       5      $ )z1Count the total number of words in the documents.c              3   Z   #    U  H!  n[        US    R                  5       5      v   M#     g7f)r!   NrN   rQ   s     r   rS   $count_total_words.<locals>.<genexpr>l   s&     @4Cs3~&,,.//4s   )+rU   )r:   s    r   count_total_wordsrb   j   s    @4@@@r   c                     [        USS9nUR                  U  Vs/ s H  o3R                  PM     sn5      n[        S5        U$ s  snf )z@Splits the text into semantic chunks using the given embeddings.
percentile)breakpoint_threshold_typez%Documents split into semantic chunks.)r
   create_documentsr!   r3   )r:   rI   r>   r@   rH   s        r    split_text_with_semantic_chunkerrg   n   sK    #lM ..D/QDS0@0@D/QRI	
12 0Rs   Ac                    [         R                  R                  U5      (       d  [         R                  " U5        [	        U 5       Hj  u  p#SUS-    S3n[         R                  R                  X5      n[        USSS9 nUR                  UR                  5        SSS5        [        SU 35        Ml     g! , (       d  f       N!= f)	zBSaves each document in the documents list as a separate .txt file.document_part_r   r   wr   )encodingNzSaved: )
r#   r$   existsmakedirs	enumerater/   openwriter!   r3   )rH   
output_dirir\   	file_namer   files          r   save_documents_to_txtru   x   s    77>>*%%
J +$QqSE.	GGLL7	)S73tJJx,,- 4 		{#$ , 43s   >B55
C	c           	         [        U5      n[        U5      n[        R                  R                  SX5      n[        R                  " USS9  [        SS9n[        U SS9 H  u  pV[        R                  R                  U5      n[        R                  " USS9  [        R                  " U/US9n[        R                  R                  US	U 35      n	UR                  U	5        [        S
U SU SU 35        M     g )Nmy_embeddingsT)exist_oknomic-embed-textrC   r   )start)rF   faiss_indexz(Saved FAISS embedding for document part z as faiss_indexz in )r5   r#   r$   r/   rm   r   rn   r   from_documents
save_localr3   )
split_documents	client_idreference_idembedding_folder_baserI   idxr@   embedding_foldertemp_dbembedding_file_paths
             r   create_and_save_embeddingsr      s    II|$LGGLL)Q KK%5 "(:;J oQ777<<(=> 	$t4 &&u
C !ggll+;{3%=PQ./8_SEQUVfUghi 8r   c                    [        SS9nS nU SU  SU 3n[        R                  " U5       Vs/ s H@  nUR                  S5      (       d  M  U[	        S5      S  R                  5       (       d  M>  UPMB     nn[        US S9nU H  n[        R                  R                  XV5      n	[        SU	 35        [        R                  " XSS	9n
U
R                  R                  5        Vs/ s H(  nU
R                  R                  U5      R                   PM*     nnUc  U
nM  UR#                  U5        M     Ub  UR%                  U S
35        [        U5        U HG  n[        R                  R                  XV5      n	 [&        R(                  " U	5        [        SU	 35        MI     U$ s  snf s  snf ! [*         a    [        SU	 35         Mr  [,         a  n[        SU	 SU 35         S nAM  S nAff = f)Nry   rC   /r{   c                 8    [        U R                  SS5      5      $ )Nr{    )intr   )xs    r   <lambda>!merge_all_faiss.<locals>.<lambda>   s    QYY}VX=Y9Zr   )keyzLoading FAISS index from: T)allow_dangerous_deserializationz/merged_faisszDeleted FAISS index folder: zFolder not found: zError deleting z: )r   r#   listdir
startswithrO   isdigitsortedr$   r/   r3   r   
load_localindex_to_docstore_idvaluesdocstoresearchr!   	add_textsr}   shutilrmtreeFileNotFoundErrorOSError)r   r   	base_pathrI   merged_faissfolder_pathfolderfaiss_folderssorted_folders
faiss_pathcurrent_faissdoc_idcurrent_textses                 r   merge_all_faissr      s   &89JL Kq1\N;K  ZZ446]+ 	06s=7I7J0K0S0S0U 	4   M/Z[N !WW\\+6
*:,78((aef (5'I'I'P'P'RT'RV '//66v>KK'R 	 T (L ""=1 !  ;-} =>	, !WW\\+6
	7MM*%0=> ! QT* ! 	5&zl34 	7OJ<r!566	7s5   F#F#/F#%/F(9$F--G-	G-G((G-c                     [         R                   " 5       n[        SU 35        [        U 5      n[        U5      n[	        SS9n[        XW5      n[        X5        [        XU5        SU SU 3n	 [        X#5      n
g )NzStart Time: ry   rC   zmy_embeddings/r   )	timer3   rY   r^   r   rg   ru   r   r   )r   rq   r   r   
start_timer:   Total_countrI   r~   embedding_dirmerge_embeddingss              r   mainr      s    J	L
%& ID+D1K ");<J 7tHO /6 <H %YKq?MT
 'y>r   __main__z	cyber.pdfztemp/output_embeddingsi  iy  )ry   )rw   )%r   r#   langchain_ollamar    langchain_community.vectorstoresr   $langchain_community.document_loadersr   r   r   langchain_text_splittersr   r   r	   r(   r   $langchain_experimental.text_splitterr
   r   r   r0   rK   rW   rY   r^   rb   rg   ru   r   r   r   __name__r   rq   r   r   r    r   r   <module>r      s     	 - 2 X X C    @ 
.`6A%j>1hH.!?F zI$J/ILJ	,7 r   