o
    W¡i ,  ã                   @   sB   d dl Z d dlZd dlZd dlZd dlmZmZ G dd„ dƒZdS )é    N)ÚListÚTuplec                   @   sà   e Zd Zddedefdd„Zdedefd	d
„Zdedefdd„Zdededeee	f fdd„Z
ddedededeee	f fdd„Zd dededefdd„Zdedefdd„Zd!dee dedee deee e	f fdd„ZdS )"Ú
Translatorúhttp://localhost:11434ú
gemma3:12bÚ
ollama_urlÚmodelc                 C   s   || _ || _|› d| _d S )Nz/api/generate)r   r   Úapi_endpoint)Úselfr   r   © r   úQ/var/www/eduai.edurigo.com/language_translate_text/production/hindi_translator.pyÚ__init__   s   zTranslator.__init__ÚtextÚreturnc                 C   s.   t  dd|¡}t |¡}t  dd|¡}| ¡ S )zUClean input text by removing HTML tags, unescaping entities, and markdown code blocksz```Ú ú<[^>]+>)ÚreÚsubÚhtmlÚunescapeÚstrip©r
   r   r   r   r   Ú_clean_input   s   
zTranslator._clean_inputc                 C   s^   d}t  ||¡}g }|D ] }|sqt  d|¡st  d|¡r%| d|f¡ q| d|f¡ q|S )z‹Split text into HTML tags and translatable text segments.

        Returns a list of tuples: ('tag', content) or ('text', content)
        z-(<[^>]+>|&[a-zA-Z]+;|&#\d+;|&#x[0-9a-fA-F]+;)z	^<[^>]+>$z)^&[a-zA-Z]+;$|^&#\d+;$|^&#x[0-9a-fA-F]+;$Útagr   )r   ÚsplitÚmatchÚappend)r
   r   ÚpatternÚpartsÚsegmentsÚpartr   r   r   Ú_extract_segments   s   zTranslator._extract_segmentsÚtarget_languagec                 C   sÜ  d|› d|› }| j |ddddg d¢dœd	œ}tj| j|d
didd}|jdkrç| ¡ }| dd¡ ¡ }| j|dd}| ¡  	| ¡ ¡rR| ¡ t
| ¡ ƒd…  ¡ }| ¡ | ¡ ks^| ¡ sÖd|› d|› }	| ¡ }
|	|
d< tj| j|
d
didd}|jdkrÖ| ¡ }| dd¡ ¡ }| j|dd}| ¡  	| ¡ ¡r©| ¡ t
| ¡ ƒd…  ¡ }| dd¡| dd¡ | dd¡| dd¡ dœ}| ¡ | ¡ ksÎ| ¡ sÒ||fS ||fS | dd¡| dd¡dœ}||fS |dddœfS )z+Translate plain text (no HTML) via the LLM.zTranslate this text to zL. Provide only the translated text, no explanations or additional content:

Fgš™™™™™É?gÍÌÌÌÌÌì?iÐ  )Ú	IMPORTANTzInstructions:zText to translate:)ÚtemperatureÚtop_pÚ
max_tokensÚstop)r   ÚpromptÚstreamÚoptionszContent-Typezapplication/jsoné<   )ÚjsonÚheadersÚtimeoutéÈ   Úresponser   )Úpreserve_htmlNz^. Translate everything, including proper names and titles. Provide only the translated text:

r(   Úprompt_eval_countr   Ú
eval_count©r2   r3   )r   ÚrequestsÚpostr	   Ústatus_coder,   Úgetr   Ú_clean_responseÚ
startswithÚlenÚcopy)r
   r   r"   r(   Úpayloadr0   ÚresultÚtranslated_textÚcleanedÚprompt_retryÚpayload_retryÚresponse_retryÚresult_retryÚtoken_usager   r   r   Ú_translate_plain_text*   sj   þüüü
þü
þ

þz Translator._translate_plain_textÚHindiFr1   c                 C   s–  z8|r(t  d|¡r|  |¡}g }g }t|ƒD ]\}\}}	|dkr1|	 ¡ r1| |	¡ | |¡ q|s<|dddœfW S d |¡}
|  |
|¡\}}| d¡}dd„ |D ƒ}t	|ƒt	|ƒkrÎ| 
d	d¡| 
d
d¡dœ}t|ƒD ]P\}}|  || |¡\}}|| d }|d t	|ƒt	| ¡ ƒ … }|t	| ¡ ƒd … }d|| ¡  | f||< |d	  | 
d	d¡7  < |d
  | 
d
d¡7  < qnd dd„ |D ƒ¡}||fW S t|ƒD ]7\}}|t	|ƒk r	|| d }|d t	|ƒt	| ¡ ƒ … }|t	| ¡ ƒd … }d|||  ¡  | f||< qÒd dd„ |D ƒ¡}||fW S |}|  ||¡\}}||fW S |  |¡}|  ||¡\}}||fW S  tyJ   |dddœf Y S w )Nr   r   r   r4   z
|||SEPARATOR|||
z|||SEPARATOR|||c                 S   s   g | ]}|  ¡ ‘qS r   )r   )Ú.0Úpr   r   r   Ú
<listcomp>‰   s    z-Translator.translate_text.<locals>.<listcomp>r2   r3   é   r   c                 s   ó    | ]\}}|V  qd S ©Nr   ©rH   Ú_Úcontentr   r   r   Ú	<genexpr>˜   ó   € z,Translator.translate_text.<locals>.<genexpr>c                 s   rL   rM   r   rN   r   r   r   rQ   £   rR   )r   Úsearchr!   Ú	enumerater   r   ÚjoinrF   r   r;   r8   ÚlstripÚrstripr   Ú	Exception)r
   r   r"   r1   r   Ú
text_partsÚtext_indicesÚidxÚseg_typerP   Úcombined_textÚtranslated_combinedrE   Útranslated_partsÚtotal_tokensÚiÚpart_translatedÚpart_tokensÚoriginalÚleadingÚtrailingr>   Úcleaned_textÚ
translatedr   r   r   Útranslate_texto   s^   


€


€



ÿzTranslator.translate_textc                 C   s~   |st  |¡}t dd|¡}g d¢}|D ]}tj|d|tjtjB d}qt dd|¡}t dd|¡}t dd	|¡}| ¡ }|S )
zBClean up LLM response to remove instruction bleeding and artifactsr   r   )z#IMPORTANT INSTRUCTIONS?:.*?(?=\n|$)zInstructions?:.*?(?=\n|$)zText to translate:.*?(?=\n|$)zHindi translation:.*?(?=\n|$)z![A-Za-z]+ translation:.*?(?=\n|$)zHere's the.*?(?=\n|$)z keeping the HTML tags.*?(?=\n|$)z\*\*.*?\*\*zExplanation:.*?(?=\n|$)z\* .*?(?=\n|$)z- Keep all HTML.*?(?=\n|$)z- Only translate.*?(?=\n|$)z- Preserve.*?(?=\n|$)z- If there are.*?(?=\n|$)z<[^>]*?\s+[^>]*?>zOriginal:.*?(?=\n|$)z'So, if the original text was.*?(?=\n|$)z+The translated version would be:.*?(?=\n|$)z9ye word mera name variable mia add ho raha hai.*?(?=\n|$))Úflagsz<\s+([^>]+)\s*>z<\1>z<([^>]+)\s+>z\n\s*\n\s*\nz

)r   r   r   r   Ú
IGNORECASEÚ	MULTILINEr   )r
   r   r1   Úcleanup_patternsr   r   r   r   r9   ²   s   
zTranslator._clean_responsec                 C   s   |sdS t |ƒd S )zs
        Estimate token count for given text
        Basic estimation: ~4 characters per token for English
        r   é   )r;   r   r   r   r   Úcount_tokensÜ   s   zTranslator.count_tokensNÚ	sentencesÚpreserve_html_listc                 C   s‚   g }d}d}t |ƒD ]*\}}|r|| nd}	|  |||	¡\}
}| |
¡ || dd¡7 }|| dd¡7 }q
|||| dœ}||fS )Nr   Fr2   r3   )Úinput_tokensÚoutput_tokensr`   )rT   ri   r   r8   )r
   rp   r"   rq   Útranslated_sentencesÚtotal_prompt_tokensÚtotal_completion_tokensra   ÚsentenceÚp_htmlrh   rE   Útoken_summaryr   r   r   Útranslate_arrayå   s   
ýzTranslator.translate_array)r   r   )rG   F)F)rG   N)Ú__name__Ú
__module__Ú__qualname__Ústrr   r   Úlistr!   ÚtupleÚdictrF   Úboolri   r9   Úintro   r   rz   r   r   r   r   r      s    
$EC*4	r   )r   r5   r,   r   Útypingr   r   r   r   r   r   r   Ú<module>   s    