
    jY              
       F   S SK Jr  S SK r S SKrS SKJr  S SKJrJrJrJ	r	J
r
  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S SKrS S
KJr  S SKJr  S SKJr  S SKJr  S SKrS SKrS SKrS SKrS SK r S SK!J"r"  \" 5       r#S SK$J%r%J&r&J'r'J(r(J)r)J*r*  S SK+J,r,J-r-  S SK.J/r/J0r0  \#Rc                  S5      S 5       r2  " S S\5      r3 " S S\5      r4S r5\#Rm                  S\3S\40S\40S.S9\	" S5      \	" S5      \	" S5      4S\7S\7S \74S! jj5       r8 \#Rm                  S"\3S\40S\40S.S9\	" S5      \	" S5      \	" S5      4S\7S\7S \74S# jj5       r9 \#Rm                  S$\3S\40S\40S.S9\	" S5      \	" S5      \	" S5      4S\7S\7S \74S% jj5       r: S SK;r;\#Rm                  S&5      \	" S5      \	" S5      4S \7S\74S' jj5       r< S S(K=J=r=   " S) S*\5      r>\#Rm                  S+5      \	" S5      \	" S5      \	" S5      \	" S5      4S\?S,\?S \?S-\?4S. jj5       r@ S S/KAJArA  \#Rm                  S05      \	" S5      \	" S5      \	" S5      \	" S5      4S\?S,\?S \?S-\?4S1 jj5       rB S S2KCJCrC  \#Rm                  S35      \	" S5      \	" S5      \	" S5      \	" S5      4S\?S,\?S \?S-\?4S4 jj5       rD  S S5KEJFrF   " S6 S7\5      rG " S8 S\5      r4\#Rm                  S9\GS\40S\40S.S9\	" S5      \	" S5      4S\7S:\74S; jj5       rH S S<KIJJrJJKrKJLrLJMrMJNrN   " S= S>\5      rO " S? S@\5      rPSA rQ  \#Rm                  SB\OS\P0S\P0S.S9\	" S5      \	" S5      \	" S5      4S\7S\7S \74SC jj5       rR S SDKSJSrS   " SE S*\5      r>\#Rm                  SF5      \	" S5      \	" S5      \	" S5      \	" S5      4S\?S,\?S \?S-\?4SG jj5       r@ g)H    )NamedTemporaryFileN)List)FastAPI
UploadFileFileFormBody)JSONResponse)Document)RecursiveCharacterTextSplitter)	BaseModel)OllamaEmbeddings)AudioSegment)r   )HTTPException)BytesIO)urlparse)load_pdf split_text_with_semantic_chunkercount_tokens_in_documentssave_documents_to_txtcreate_and_save_embeddingsmerge_all_faiss)load_doccount_tokens_in_doc_documents)load_pptcount_tokens_in_ppt_documentsz)/helloEdurigoAiChatAndGenQuestionsTestingc                  
    SS0$ )Nmessagez!Hello from edurigo_ai testing env r       U/var/www/eduai.edurigo.com/question_generate/production/question_generate_fast_api.pyhellor"      s    :;;r    c                   *    \ rS rSr% \\S'   \\S'   Srg)TrainResponse'   r   token_countr   N__name__
__module____qualname____firstlineno__str__annotations__int__static_attributes__r   r    r!   r$   r$   '       Lr    r$   c                        \ rS rSr% \\S'   Srg)ErrorResponse+   errorr   Nr(   r)   r*   r+   r,   r-   r/   r   r    r!   r2   r2   +       Jr    r2   c                 .    [         R                  " SSSSSU /SSS9nUR                  $ ! [         R                   aV  nUR                  (       a(  UR                  R                  SS	S
9R                  5       O
[        U5      n[        SSU 3S9eS nAff = fNcurlz-kz-sz-Lz-fT)capture_outputcheckutf-8ignore)errors  zError fetching document: status_codedetail	
subprocessrunstdoutCalledProcessErrorstderrdecodestripr,   r   urlresulte
stderr_msgs       r!   fetch_document_with_curlrP   /       ^tT4 EVZbfg}}(( ^JK((QXX__WX_>DDFX[\]X^
6OPZ|4\]]^   '* BABBz/train_with_document_edurigo_aimodel)r?     )response_model	responses.	client_iddocument_urlreference_idc                 
   [        U5      R                  n[        R                  R                  U5      S   R	                  5       nUS;  a
  [        SSS9e[        R                  " 5       nS n [        R                  " 5       n[        U5      n[        S[        R                  " 5       U-
  S S35        [        R                  " 5       n	[        R                  " S	[        R                  R                  U5      S   S
9 n
U
R                  U5        U
R                  nS S S 5        [        S[        R                  " 5       U	-
  S S35        [        R                  " 5       nUS:X  a  S	n [        US5       nUR                  S5      nSU;   a  SnS S S 5        U(       a  [!        U5      nOSn [        USSSS9 nUR                  S5      R#                  5       nS S S 5        UR	                  5       R%                  S5      (       a
  [        SSS9eU(       a1  [        SUS S < S35        SSKJn  U" USS9nUR+                  5       nO8[        SS S9eUS!:X  a  SSKJn  U" USS9nUR+                  5       nO
[        SS"S9e[        S#[        R                  " 5       U-
  S S$[-        U5       35        [        R                  " 5       n[/        S%S&9n[1        UU5      n[        S'[        R                  " 5       U-
  S S([-        U5       35        [        R                  " 5       nS)U  S*U 3n[3        UU5        [        S+[        R                  " 5       U-
  S S35        [        R                  " 5       n[5        UX5        [        S,[        R                  " 5       U-
  S S35        [        R                  " 5       n[7        X5      n[        S-[        R                  " 5       U-
  S S35        [        R                  " 5       n[9        U5      n[        S.[        R                  " 5       U-
  S S35        [        S/[        R                  " 5       U-
  S S35        S0US1.U(       a<  [        R                  R;                  U5      (       a  [        R<                  " U5        $ $ $ ! , (       d  f       GN= f! , (       d  f       GN%= f! [         a  n[        SU 35         S nAGNFS nAff = f! , (       d  f       GN= f! [         a  n S nAGN&S nAff = f! [         a  n[        S2S3U 3S9eS nAff = f! U(       a<  [        R                  R;                  U5      (       a  [        R<                  " U5        f f f = f)4N   ).pdf.txt.docx.doc.pptx.pptr?   zRUnsupported file type. Only .pdf, .txt, .docx, .doc, .pptx and .ppt are supported.r@   z[TIME] Download: .2fsFdeletesuffixz[TIME] Save temp file: r\   rbi   s   %PDFTzError reading file header:  rr<   r=   )encodingr>   i   )z<htmlz	<!doctypez<divz<headz<bodyzThe downloaded file is an HTML page (possibly an error page, login page, or redirect) instead of a valid PDF document. Please verify the URL.zCWarning: File extension is .pdf but content is text starting with: 2   z. Falling back to TextLoader.r   )
TextLoader)rj   z?The file is empty or could not be read as a valid PDF document.r]   zUnsupported file typez[TIME] Load document: z
s | Docs: nomic-embed-textrS   z[TIME] Split text: s | Chunks: temp/_z[TIME] Save text: z[TIME] Embeddings: z[TIME] Merge FAISS: z[TIME] Token count: z[TOTAL TIME]: z7Training and embedding creation completed successfully.r   r&   rT   Processing failed: )r   pathossplitextlowerr   timerP   printtempfiler   writenameopenread	Exceptionr   rJ   
startswith$langchain_community.document_loadersrl   loadlenr   r   r   r   r   r   existsremove)rW   rX   rY   rt   file_extensionstart_totaltemp_file_patht1document_contentt2	temp_filet3is_pdffheaderrN   docscontent_previewrl   loadert4
embeddingssplit_documentst5
output_dirt6t7merged_embeddingst8r&   s                                 r!   process_documentr   8   s    L!&&DWW%%d+A.446NOO  5I  J  	J))+KNs*YY[3LA!$))+"23!7q9: YY[((bgg>N>N|>\]^>_`dmOO,-&^^N a 	'		b(8'=Q?@
 YY[V#F9.$/1VVD\F&(!% 0 / #%ncGHUYZ*+&&,*<*<*> V #((*556fgg'$'  o  %_`opsqs`t_w  xU  V  WO'IF!;;=D'$'` 
 v%GAF;;=D  C8OPP&tyy{R'7&<Js4ykRS YY[%,>?
:4L 	#DIIK"$4S#9c/FZE[\] YY[YKq7
oz:"499;#3C"8:; YY["?IL#DIIK"$4S#9;< YY[+ID$TYY[2%5c$:!<= YY[/5$TYY[2%5c$:!<=tyy{[8=Q?@Tdop rww~~n==		.)  >NG a` 0/  93A37889 VU  B  O6I!4MNNO rww~~n==		.)  >Ns   .BT >RAT (R> 4R,R> T ,S6 8 S$S6  I6T 
R)$T ,
R;6R> 9T ;R> >
S!ST S!!T $
S3.S6 1T 3S6 6
T T TT 
T*T%%T**T- -AU2z/train_with_doc_edurigo_aic                    [         R                  R                  U5      S   R                  5       nUS;  a
  [	        SSS9e[
        R
                  " 5       nSn [
        R
                  " 5       n[        U5      n[        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n[        R                  " S
US9 n	U	R                  U5        U	R                  nSSS5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n
[        U5      n[        S[
        R
                  " 5       U
-
  S S[        U5       35        U(       d
  [	        SSS9e[
        R
                  " 5       n[        SS9n[        X5      n[        S[
        R
                  " 5       U-
  S S[        U5       35        [
        R
                  " 5       nSU  SU 3n[!        UU5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n[#        XU5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n[%        X5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n['        U5      n[        S[
        R
                  " 5       U-
  S S	35        [        S[
        R
                  " 5       U-
  S S	35        SUS.U(       a<  [         R                  R)                  U5      (       a  [         R*                  " U5        $ $ $ ! , (       d  f       GN= f! [         a    e [,         a  n[	        SSU 3S9eSnAff = f! U(       a<  [         R                  R)                  U5      (       a  [         R*                  " U5        f f f = f)z.Train embeddings from DOC/DOCX document files.r[   )r_   r^   r?   z9Unsupported file type. Only .doc and .docx are supported.r@   Nz[DOC] [TIME] Download: rb   rc   Frd   z[DOC] [TIME] Save temp file: z[DOC] [TIME] Load document: s | Documents: z5No text content could be extracted from the document.rm   rn   z[DOC] [TIME] Split text: ro   rp   rq   z[DOC] [TIME] Save text: z[DOC] [TIME] Embeddings: z[DOC] [TIME] Merge FAISS: z[DOC] [TIME] Token count: z[DOC] [TOTAL TIME]: z@DOC/DOCX training and embedding creation completed successfully.rr   rT   rs   )ru   rt   rv   rw   r   rx   rP   ry   rz   r   r{   r|   r   r   r   r   r   r   r   r   r   r   r   rW   rX   rY   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r&   rN   s                         r!   process_document_docr      s0    WW%%l3A6<<>N..4opp))+KN:&YY[3LA'		b(8'=Q?@ YY[((nMQZOO,-&^^N N 	-diikB.>s-C1EF YY[',TYY[2-=c,B/RUVZR[Q\]^C8opp YY[%,>?
:4L)$))+*:3)?|CP_L`Kabc YY[YKq7
oz:(r)9#(>a@A YY["?|L)$))+*:3)?qAB YY[	0*499;+;C*@BC YY[3D9*499;+;C*@BC$TYY[;%>s$C1EF]nyz bggnn^<<IIn% =>c NMX   O6I!4MNNO bggnn^<<IIn% =>?   A0M3 	M!'H6M3 !
M0+M3 3NNNN AO!z/train_with_ppt_edurigo_aic                    [         R                  R                  U5      S   R                  5       nUS;  a
  [	        SSS9e[
        R
                  " 5       nSn [
        R
                  " 5       n[        U5      n[        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n[        R                  " S
US9 n	U	R                  U5        U	R                  nSSS5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n
[        U5      n[        S[
        R
                  " 5       U
-
  S S[        U5       35        U(       d
  [	        SSS9e[
        R
                  " 5       n[        SS9n[        X5      n[        S[
        R
                  " 5       U-
  S S[        U5       35        [
        R
                  " 5       nSU  SU 3n[!        UU5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n[#        XU5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n[%        X5        [        S[
        R
                  " 5       U-
  S S	35        [
        R
                  " 5       n['        U5      n[        S[
        R
                  " 5       U-
  S S	35        [        S[
        R
                  " 5       U-
  S S	35        SUS.U(       a<  [         R                  R)                  U5      (       a  [         R*                  " U5        $ $ $ ! , (       d  f       GN= f! [         a    e [,         a  n[	        SSU 3S9eSnAff = f! U(       a<  [         R                  R)                  U5      (       a  [         R*                  " U5        f f f = f)z2Train embeddings from PPT/PPTX presentation files.r[   )ra   r`   r?   z9Unsupported file type. Only .ppt and .pptx are supported.r@   Nz[PPT] [TIME] Download: rb   rc   Frd   z[PPT] [TIME] Save temp file: z [PPT] [TIME] Load presentation: r   z9No text content could be extracted from the presentation.rm   rn   z[PPT] [TIME] Split text: ro   rp   rq   z[PPT] [TIME] Save text: z[PPT] [TIME] Embeddings: z[PPT] [TIME] Merge FAISS: z[PPT] [TIME] Token count: z[PPT] [TOTAL TIME]: z@PPT/PPTX training and embedding creation completed successfully.rr   rT   rs   )ru   rt   rv   rw   r   rx   rP   ry   rz   r   r{   r|   r   r   r   r   r   r   r   r   r   r   r   r   s                         r!   process_document_pptr     s0    WW%%l3A6<<>N..4opp))+KN:&YY[3LA'		b(8'=Q?@ YY[((nMQZOO,-&^^N N 	-diikB.>s-C1EF YY['0r1A#0FoVYZ^V_U`abC8stt YY[%,>?
:4L)$))+*:3)?|CP_L`Kabc YY[YKq7
oz:(r)9#(>a@A YY["?|L)$))+*:3)?qAB YY[	0*499;+;C*@BC YY[3D9*499;+;C*@BC$TYY[;%>s$C1EF]nyz bggnn^<<IIn% =>c NMX   O6I!4MNNO bggnn^<<IIn% =>r   z/untrain_document_edurigo_aic                   #     SU 3n[         R                  R                  X 5      n[         R                  R                  U5      (       aU  [         R                  R	                  U5      (       a  [         R
                  " U5        O[        R                  " U5        SS0$ [        SSS9e! [         a  n[        S[        U5      S9eS nAff = f7f)Nzmy_embeddings/r   zDocument untrained successfullyi  zDocument not foundr@   rT   )ru   rt   joinr   isfiler   shutilrmtreer   r   r,   )rY   rW   
local_path	file_pathrN   s        r!   delete_documentr   T  s     <%i[1
GGLL:	77>>)$$ww~~i((		)$i(@AAC8LMM <CF;;<s/   CBB+  C!
B+ +
C5CCCgenerate_mcqc                   >    \ rS rSr% \\S'   \\S'   \\S'   \\S'   Srg)GenerateMCQRequestii  rW   num_questionsrY   isGPUr   Nr(   r)   r*   r+   r.   r-   r/   r   r    r!   r   r   i      N
Ir    r   z/generate_mcqr   r   c                 B   #    [        XX#5      I S h  vN u  pEXTS.$  N
7f)Nhistory_countmcqsr   rW   r   rY   r   r   r   s         r!   generate_mcq_endpointr   o  s&      ,Y| [[D*99 \   generate_mcq_docz/generate_mcq_docc                 B   #    [        XX#5      I Sh  vN u  pEXTS.$  N
7f)z8Generate MCQs from trained DOC/DOCX document embeddings.Nr   r   r   s         r!   generate_mcq_doc_endpointr   y  (      !1< __D*99 `r   generate_mcq_pptz/generate_mcq_pptc                 B   #    [        XX#5      I Sh  vN u  pEXTS.$  N
7f)z<Generate MCQs from trained PPT/PPTX presentation embeddings.Nr   r   r   s         r!   generate_mcq_ppt_endpointr     r   r   )answer_questionc                   D    \ rS rSr% \\S'   \\S'   \\S'   \\   \S'   Sr	g)ChatResponsei  answerprocessing_timetotal_tokenssource_documentsr   N)
r(   r)   r*   r+   r,   r-   floatr.   r   r/   r   r    r!   r   r     s    K3ir    r   c                        \ rS rSr% \\S'   Srg)r2   i  r4   r   Nr5   r   r    r!   r2   r2     r6   r    z/chat_with_edurigo_aiquestionc                    #     [        [        U 5      U5      u  p#pE[        UUUUS9$ ! [         a  n[	        S[        U5      S9eS nAf[         a  n[	        SS[        U5       3S9eS nAff = f7f)N)r   r   r   r   r?   r@   rT   zAn error occurred: )r   r.   r   FileNotFoundErrorr   r,   r   )rW   r   r   source_reference_idsr   r   rN   s          r!   chat_with_documentr     s     TFUVYZcVdfnFoCl+%1	
 	
  <CF;; T6I#a&4RSSTs0   A0#( A0
A-AA-A((A--A0)convert_video_to_audioaudio_to_textload_and_split_document1save_faiss_indexcount_tokens_videoc                   *    \ rS rSr% \\S'   \\S'   Srg)TrainResponse1i  r   r&   r   Nr'   r   r    r!   r   r     r0   r    r   c                        \ rS rSr% \\S'   Srg)ErrorResponse1i  r4   r   Nr5   r   r    r!   r   r     r6   r    r   c                 .    [         R                  " SSSSSU /SSS9nUR                  $ ! [         R                   aV  nUR                  (       a(  UR                  R                  SS	S
9R                  5       O
[        U5      n[        SSU 3S9eS nAff = fr8   rC   rK   s       r!   fetch_document_with_curl1r     rQ   rR   z/train_with_video_edurigo_aic                 f  #    [         R                  R                  U5      S   R                  5       nUS;  a
  [	        SSS9eSU 3n SU S3nU S	3nS
U  SU 3n[        U5      n[        US5       n	U	R                  U5        S S S 5        Sn
U
S:  a
  [	        SSS9e[        XE5        [        XV5      n[        U5      nU(       d
  [	        SSS9e[        U5      nS
U  SU 3n[         R                  " USS9  [        X5        SUS.[         R                  R                  U5      (       a  [         R                  " U5        [         R                  R                  U5      (       a  [         R                  " U5        $ $ ! , (       d  f       N= f! [          a  n[	        SS[#        U5       3S9eS nAff = f! [         R                  R                  U5      (       a  [         R                  " U5        [         R                  R                  W5      (       a  [         R                  " U5        f f = f7f)Nr[   )z.mp4r?   z+Unsupported file type. Only .mp4 supported.r@   temp_documentz
mp3-files/z.mp3r]   zmy_embeddings_video//wb      zVideo file too large.zNo valid documents to process.T)exist_okSuccessrr   rT   rs   )ru   rt   rv   rw   r   r   r}   r{   r   r   r   r   makedirsr   r   r   r   r,   )rW   rX   rY   r   r   mp3_output_pathoutput_filefaiss_save_pathr   r   video_durationtextfinal_documentsr&   folder_pathrN   s                   r!   process_document_videor     s    WW%%l3A6<<>NX%4abb$^$45N)'&|nD9%d+01\NK 5\B.$'9OO,- ( BC8OPP~?_: 3;?C8XYY(9 -YKqG
K$/:$[A 77>>.))IIn%77>>/**IIo& +C ('8  T6I#a&4RSST 77>>.))IIn%77>>/**IIo& +sQ   AH1	*F 3E>BF A6H1>
FF 
F4F//F44F7 7A7H..H1generate_mcq_videoc                   >    \ rS rSr% \\S'   \\S'   \\S'   \\S'   Srg)r   i  rW   r   rY   r   r   Nr   r   r    r!   r   r     r   r    z/generate_mcq_videoc                 F   #    [        XX#5      I S h  vN u  pEnXTUS.$  N7f)N)r   r   r&   r   )rW   r   rY   r   r   r   r&   s          r!   r   r     s+     -?	Zf-n'n$D*UU (os   !!)Trz   r   	tracebacktypingr   fastapir   r   r   r   r	   fastapi.responsesr
   langchain_core.documentsr   langchain_text_splittersr   pydanticr   langchain_ollamar   uvicornpydubr   fastapi.datastructuresr   ior   rD   ru   asynciorequestsrx   urllib.parser   apptrainingr   r   r   r   r   r   training_docr   r   training_pptr   r   getr"   r$   r2   rP   postr,   r   r   r   r   r   r   r   r.   r   r   r   r   r   chattingr   r   r   training_videor   r   r   r   r   r   r   r   r   r   r   r    r!   <module>r
     s   '    9 9 * - C  -   - !   	    !i _  _ @ @ 	45< 6< <I I ^ 
+M\cer[s  |C  ER  {S  VT  U&*3iT#Ydhildm @* @*s @*^a @* U@*F @
&}W^`mVnv}  @M  vN  QO  P*.s)chlmphq D&C D&3 D&be D& PD&N D
&}W^`mVnv}  @M  vN  QO  P*.s)chlmphq C&C C&3 C&be C& PC&L  
().23i$s) < <C < *<   %  /15cQUVYQZptuxpy  IM  NQ  IR :3 :3 :jm :  CF : :
 . -
59#YUYZ]U^txy|t}  MQ  RU  MV :s :s :nq :  GJ : : 2 -
59#YUYZ]U^txy|t}  MQ  RU  MV :s :s :nq :  GJ : :   $ 9  I  
!,QXZgPhpw  zG  pH  KI  J.23ic T T3 T JT   ~ ~Y Y ^[G 
(ZacqYr  {B  DR  zS  TT  U26s)QUVYQZptuxpy 0'C 0'3 0'jm 0' U0'f # 1  
 15cQUVYQZptuxpy  IM  NQ  IR V3 V3 Vjm V  CF V !V ;Mr    