
    Mi              	      8   % S SK Jr  S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
rS SKrS SKJr  S SKJrJrJr  S SKrS SKJr  \R,                  " \5      rSrS\S'   S	rS
\S'   SrS
\S'   S\ S\ S\ S\ S3	rS
\S'   SrS
\S'   SrS\S'   \R@                  RC                  \	RD                  " 5       \ S35      r#S
\S'   S%S jr$S&S jr%/ SQr&S'S jr'\" SS9S'S j5       r(S(S  jr)S)S! jr*\" \S9S)S" j5       r+\" \S9S*S# j5       r,\" \S9S+S$ j5       r-g),    )annotationsN)	lru_cache)FinalListTuple)FileLock   z
Final[int]CACHE_MAX_SIZEen_core_web_smz
Final[str]_SPACY_MODEL_NAMEz3.8.0_SPACY_MODEL_VERSIONz<https://github.com/explosion/spacy-models/releases/download/-/-py3-none-any.whl_SPACY_MODEL_URL@1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85_SPACY_MODEL_SHA256x   _DOWNLOAD_TIMEOUT_SECONDSz.install.lock_INSTALL_LOCK_PATHc                h    [         R                  R                  U [        S9 n[	        US5       n[
        R                  " X#5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f! [         R                  R                   a  n[        SU  SU S35      UeSnAff = f)z;Download a URL to a local file with a socket-level timeout.timeoutwbNz$Failed to download spaCy model from : z.. Check your network connection and try again.)
urllibrequesturlopenr   openshutilcopyfileobjerrorURLErrorRuntimeError)urldestrespoutexcs        s/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/unstructured/nlp/tokenize.py_download_with_timeoutr+   '   s    ^^##C1J#KtdD!S""4- " LK!! LK <<   23%r# ?; ;
 	sK   "A; A*AA*A; 
A'	#A**
A84A; 8A; ;B1B,,B1c                    SSK Jn   SSKJn  SSKJn  SSKJn  [        R                  " 5        n[        R                  R                  U[         S[         S35      n[        R!                  S[        [        5        [#        [$        U5        ['        US	5       n[(        R*                  " UR-                  5       5      R/                  5       nS
S
S
5        W[0        :w  a  [3        S[         S[0         SU 35      e[        R                  R                  US5      n[4        R6                  " 5       n	U	R9                  5       n
XS'   XS'   U" U
[:        R<                  U" 5       S9nUR'                  U5       nU " X0 S9  S
S
S
5        U	S   n[        R>                  " U5       H  n[        R                  R                  X5      n[        R                  R                  X5      n [        R                  RA                  U5      (       a  [B        RD                  " U5        O:[        R                  RG                  U5      (       a  [        RH                  " U5        [B        RJ                  " UU5        M     S
S
S
5        [        R!                  S[        [        5        g
! , (       d  f       GN= f! , (       d  f       GN6= f! [L         a%  n[3        S[         SU SU S[         35      UeS
nAff = f! , (       d  f       N= f)zPDownload and install the pinned spaCy model wheel using the `installer` library.r   )install)SchemeDictionaryDestination)	WheelFile)get_launcher_kindr   r   u!   Downloading spaCy model %s %s …rbNzHash mismatch for z: expected z, got stagingpurelibplatlib)interpreterscript_kind)sourcedestinationadditional_metadatazFailed to install z to r   zj. Ensure the site-packages directory is writable, or pre-install the model with: python -m spacy download zInstalled %s %s)'	installerr-   installer.destinationsr.   installer.sourcesr/   installer.utilsr0   tempfileTemporaryDirectoryospathjoinr   r   loggerinfor+   r   r   hashlibsha256read	hexdigestr   r$   	sysconfig	get_pathscopysys
executablelistdirisdirr    rmtreeexistsremovemoveOSError)r-   r.   r/   r0   tmpwhl_pathfrF   r2   pathsstaged_pathsr8   r7   site_packagesitemsrcdstr)   s                     r*   _install_spacy_modelr^   4   sU   !B+1		$	$	&#77<<(9':!<P;QQb%cd79JL`a/:(D!Q^^AFFH-779F "(($%6$7 8/0vh@  '',,sI.##%zz|")Y")Y1)+

 ^^H%6PRS & i(JJw'D'',,w-C'',,}3C77==%%MM#&WW^^C((IIcNC% (E 
'd KK!#46JK[ "!( &%$  "():(;4bQTPU V66G5HJ 	W 
'	&sc   A*K-3JB'K-2J):A$K-BJ;+K-
J&	!K-)
J8	3K-;
K* K%%K**K--
K;)ner
lemmatizerattribute_rulerc                     [         R                  " [        [        S9$ ! [         a     Of = f[        [        SS9   [        R                  " 5          [         R                  " [        [        S9sS S S 5        $ ! [         a     Of = f[        5         [        R                  " 5          [         R                  " [        [        S9sS S S 5        $ ! [         a  n [        S[         S35      U eS n A ff = f! , (       d  f       g = f)N)excluder   z
Installed z[ but spacy.load() still failed. Check site-packages permissions and installation integrity.)spacyloadr   _SPACY_EXCLUDErT   r   r   	importlibinvalidate_cachesr^   r$   )r)   s    r*   _load_spacy_modelrj   v   s    zz+^DD  
$b	1##%	::/H	 
2	1
  		##%	::/H 
2	1  	./ 0N N 	 
2	1sS    
,,C6A::
BC6B"C6*C
C3C..C33C66
D   )maxsizec                     [        5       $ )zOLoad the spaCy model on first use and cache it for the lifetime of the process.)rj        r*   _get_nlprp      s     ro   c                4    [        5       " [        U 5      5      $ )zVRun the spaCy pipeline once. All public functions extract what they need from the Doc.)rp   strtexts    r*   _processru      s     :c$i  ro   c                *    [        [        U 5      5      $ )zA wrapper so that we can cache the result of sentence tokenization as an
immutable, while returning the expected return type (list).)list_tokenize_for_cachers   s    r*   sent_tokenizery      s     #D)**ro   c                V    [        U 5       Vs/ s H  oR                  PM     sn$ s  snf )zCA wrapper around the spaCy word tokenizer with LRU caching enabled.)ru   rt   )rt   tokens     r*   word_tokenizer|      s"     %-TN3N5JJN333s   &c                r    [        U 5      nU Vs/ s H  o"R                  UR                  4PM     sn$ s  snf )z?A wrapper around the spaCy POS tagger with LRU caching enabled.)ru   rt   tag_)rt   docr{   s      r*   pos_tagr      s0     4.C256#ZZ$#666s   !4c                L    [        S [        U 5      R                   5       5      $ )zGA wrapper around the spaCy sentence tokenizer with LRU caching enabled.c              3  8   #    U  H  oR                   v   M     g 7f)Nrs   ).0sents     r*   	<genexpr>&_tokenize_for_cache.<locals>.<genexpr>   s     <';t';s   )tupleru   sentsrs   s    r*   rx   rx      s     <x~';';<<<ro   )r%   rr   r&   rr   returnNone)r   r   )r   zspacy.language.Language)rt   rr   r   zspacy.tokens.Doc)rt   rr   r   z	List[str])rt   rr   r   zList[Tuple[str, str]])rt   rr   r   zTuple[str, ...]).
__future__r   rE   rh   loggingr@   r    rL   rI   r>   urllib.errorr   urllib.request	functoolsr   typingr   r   r   re   filelockr   	getLogger__name__rC   r
   __annotations__r   r   r   r   r   rA   rB   
gettempdirr   r+   r^   rg   rj   rp   ru   ry   r|   r   rx   rn   ro   r*   <module>r      sm   "    	  
      % %  			8	$ 
   0 : 0#* j *B/0/00AC * 
 #e Z d ), : +!#/0>" J 

9L~ :6 1 
!+ >"4 #4
 >"7 #7 >"= #=ro   