
    Mi                         S SK r S SKrS SKrS SKrS SKrS SKJr  S SKJr  S SK	r	S SK
J
r
  S SKJr  S SKJr  \" 5       rSrS\S	\4S
 jrS\4S jrSS\S\S\4S jjrS\4S jrS\S\4S jr " S S5      rg)    N)ThreadPoolExecutor)Path)tqdm)
get_logger)settingsiX  url1url2c                 T    U R                  S5      n UR                  S5      nU  SU 3$ )N/)rstriplstrip)r   r	   s     i/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/surya/common/s3.py	join_urlsr      s/    ;;sD;;sDV1TF    pretrained_model_name_or_pathc                 *    U R                  S5      S   $ )Nr   r   )split)r   s    r   get_model_namer      s    (..s3A66r   remote_path
local_path
chunk_sizec           	         [        U5      n [        R                  " U SSS9nUR                  5         [	        UR
                  R                  SS5      5      nUR                  n[        USSSSU 3SS	9n[        US
5       nSnUR                  US9 HE  n	U	(       d  M  UR                  U	5        U[        U	5      -  nUR                  [        U	5      5        MG     S S S 5        UR                  5         U$ ! , (       d  f       N = f! [         aO  n
UR                  5       (       a  UR!                  5         ["        R%                  SU  S['        U
5       35        e S n
A
ff = f)NT)streamallow_redirectszcontent-lengthr   Bi   Downloading    )totalunit
unit_scaleunit_divisordescminiterswb)r   zDownload error for file z: )r   requestsgetraise_for_statusintheadersnamer   openiter_contentwritelenupdateclose	Exceptionexistsunlinkloggererrorstr)r   r   r   response
total_sizefilenamepbarf
downloadedchunkes              r   download_filer?      sC   j!J <<D$O!!# ))--.>BC
 ??z*
 *d#qJ!..*.E5GGEN#e*,JKKE
+	 F $ 	

 $#  /}Bs1vhGH	s7   A4D C6=C6D 6
D D 
E A
EE 	local_dirc                 p   [        U 5      n U S-  n[        R                  R                  U5      (       d  g [	        US5       n[
        R                  " U5      nS S S 5        WS    H+  n[        R                  R                  X-  5      (       a  M+    g   g! , (       d  f       NC= f! [         a     gf = f)Nmanifest.jsonFrfilesT)r   ospathr2   r+   jsonloadr1   )r@   manifest_pathr;   manifestfiles        r   check_manifestrL   D   s    YI/M77>>-((-%yy|H &W%D77>>)"233 &  &%
  s5   B( B4B( B( B( 
B%!B( (
B54B5c           
      $   [        U 5      n[        [        R                  U 5      n[	        U5      nU(       a  g [
        R                  " 5        n[        US5      n[        R                  R                  US5      n[        Xg5        [        US5       n[        R                  " U5      n	S S S 5        [        SU SU 3[        W	S   5      S9n
[!        [        R"                  S9 n/ nU	S    HR  n[        X=5      n[        R                  R                  X]5      nUR%                  UR'                  [        X5      5        MT     U H$  nUR)                  5         U
R+                  S5        M&     S S S 5        U
R-                  5         [        R.                  " U5       H7  n[0        R2                  " [        R                  R                  X]5      U5        M9     S S S 5        g ! , (       d  f       GN<= f! , (       d  f       N= f! , (       d  f       g = f)	NrB   rC   r   z
 model to rD   )r"   r   )max_workersr   )r   r   r   S3_BASE_URLrL   tempfileTemporaryDirectoryrE   rF   joinr?   r+   rG   rH   r   r.   r   PARALLEL_DOWNLOAD_WORKERSappendsubmitresultr/   r0   listdirshutilmove)r   r@   
model_names3_urlmodel_existstemp_dirmanifest_filerI   r;   rJ   r:   executorfuturesrK   remote_file
local_filefutures                    r   download_directoryrd   V   s   ,Jx++[9F!),L 
	$	$	&(!&/:X?m3 -%yy|H & 
|:i[Ahw'(

   ::
G )'5WW\\(9
x}kVW *
 "A "
 	

 JJx(DKKX4i@ )? 
'	& &%
 
 
'	&s?   AHG);H$BG0,A)H
G-	(H0
G>	:H
Hc                   N   ^  \ rS rSrSr\S\4S j5       r\U 4S j5       rSr	U =r
$ )S3DownloaderMixin   zs3://returnc                    UR                  U R                  5      (       ab  UR                  U R                  S5      n[        R                  n[
        R                  R                  X!5      n[
        R                  " USS9  U$ SnU$ )N T)exist_ok)	
startswith	s3_prefixreplacer   MODEL_CACHE_DIRrE   rF   rR   makedirs)clsr   	cache_dirr   s       r   get_local_path S3DownloaderMixin.get_local_path   ss    (33CMMBB,I,Q,Qr-) !00IiOJKK
T2  Jr   c                 r  > UR                  U R                  5      (       d  [        T
U ]  " U/UQ70 UD6$ U R	                  U5      nUR                  U R                  S5      nSnSnSnSnU(       d!  Xu:  a   [        X5        SnU(       d  Xu:  a  M  [        T
U ]  " U/UQ70 UD6$ ! [         a  n	[        R                  SU SUS	-    S
U SU	 35        US	-  nXu:  a4  [        R                  SU S35        [        R                  " U5         S n	A	N[        R                  SU SU S35        U	eS n	A	ff = f)Nrj         r   FTzError downloading model from z
. Attempt r   z of z	. Error: zRetrying in z seconds...zFailed to download z after z
 attempts.)rl   rm   superfrom_pretrainedrs   rn   rd   r1   r4   r5   infotimesleep)rq   r   argskwargsr   retriesdelayattemptsuccessr>   	__class__s             r   ry   !S3DownloaderMixin.from_pretrained   sx    -77FF7*-048>  ''(EF
(E(M(MMM2)
%
 g/"#@M g/$ w&zCDCFCC  34Q3RR\]dgh]h\iimnumvv  AB  @C  D 1$KK,ug[ ABJJu%%LL-.K-LGT[S\\fg Gs   8B' '
D61AD1D11D6 )__name__
__module____qualname____firstlineno__rm   classmethodr6   rs   ry   __static_attributes____classcell__)r   s   @r   rf   rf      s7    I
c 
 
 #D #Dr   rf   )i   )rG   rE   rX   rP   r{   concurrent.futuresr   pathlibr   r%   r   surya.loggingr   surya.settingsr   r4   LOCK_EXPIRATIONr6   r   r   r(   r?   rL   rd   rf   r   r   r   <module>r      s     	    1    $ #	 C s 7# 7"s " " "Jc $)AC )AC )AX4D 4Dr   