
    Mi                    J   S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
Jr  S SKJr  S SKJrJrJrJrJrJrJrJrJrJrJr  S SKrS SKrS SKJr  S SKJr  S SKJ r   S SK!r!S SK"r!S SK#r!S SK!J$r$  S S	K%J&r&  S S
K'J(r(  S SK)J*r*J+r+J,r,J-r-  S SK.J/r/  S SK0J1r1  S SK2J3r3  S SK4J5r5J6r6  S SK7J8r8  S SK9J:r:  S SK;J<r<  S SK=J>r?  S SK@JArA  S SKBJCrC  S SKDJErE  S SKFJGrG  S SKHJIrI  S SKJJKrK  S SKLJMrM  S SKNJOrO  S SKPJQrQ  S SKRJSrSJTrT  S SKUJVrVJWrWJXrX  S SKYJZrZ  S S K[J\r\J]r]  S S!K^J_r_  S S"K`JaraJbrbJcrcJdrdJere  S S#KfJgrgJhrhJiri  S S$KjJkrl  S S%KmJnrnJoro  S S&KpJqrq  S S'KrJsrs  S S(KtJuruJvrv  S S)KwJxrxJyry  S S*KzJ{r{J|r|J}r}J~r~  S S+KJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJrJr  S S,KJr  S S-KJrJrJr  S S.KJr  S S.KJr  S S/KJrJrJr  S S0KJrJrJrJrJrJrJrJrJrJr  S1S2KJr  \(       a  S S3KJr  S S4KJrJr  \\\4   rO\r\r\r " S5 S6\GRf                  5      r " S7 S85      rg)9    Ndefaultdict)	lru_cache)TYPE_CHECKINGAnyAsyncGeneratorCallableDictListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)uuid)	DualCacheInMemoryCache
RedisCacheRedisClusterCache)DEFAULT_MAX_LRU_CACHE_SIZE)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs&get_metadata_variable_name_from_kwargs)coroutine_checker)CredentialAccessor)tracer)Logging)SensitiveDataMasker)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2)simple_shuffle)get_deployments_for_tag) add_fallback_headers_to_responseadd_retry_headers_to_response)"_get_router_metadata_variable_namereplace_model_in_jsonlshould_replace_model_in_jsonl)InitalizeCachedClient)get_dynamic_litellm_paramsis_clientside_credential)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)PromptCachingDeploymentCheck)ResponsesApiDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValues	FileTypesOpenAIFileObjectOpenAIFilesPurpose)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCredentialLiteLLMParamsCustomRoutingStrategyBase
DeploymentDeploymentTypedDictLiteLLM_ParamsMockRouterTestingParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategySearchToolTypedDict)ServiceTypes)CustomPricingLiteLLMParamsGenericBudgetConfigTypeLiteLLMBatch)	ModelInfo)ModelResponseStreamStandardLoggingPayloadUsage)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Span)
AutoRouterPreRoutingHookResponsec                       \ rS rSrSrSrg)RoutingArgs   <    N)__name__
__module____qualname____firstlineno__ttl__static_attributes__r{       h/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/litellm/router.pyrx   rx      s    
Cr   rx   c            W          \ rS rSr% \" 5       r\\S'   Sr\\	   \S'   Sr
\\S'   SrSr\\   \S'   Sr\\   \S	'   Sr\\\\\\4         \S
'   SSSSSSSS0 SSSSSSSSSSSSS/ / / 0 SSSS0 SSSSSS0 SS\" 5       S4*S\\\\   \\\\4      4      S\\   S\\\      S\\   S\\   S\\   S\\   S\\	   S\S\\\      S\S\\   S\\   S\\   S\\   S\\   S\\   S\\   S\\   S \	S!\ S"   S#\\\      S$\S%\S&\S'\\\\\\!4   4      S(\	S)\	S*\S+\\\"\4      S,\\\"4   S-\\   S.\\#   S/\\   S0\\	   S1\ S2   S3\\$   S4\S5\\%   S6\\&   S7\\   S8\	S9S4VS: jjr'S; r(S< r)\*S=\\\4   S9\\+\,4   4S> j5       r-S?\+4S@ jr.S1\\/\4   S4\4SA jr0SB r1SC r2SD r3SE r4SF\\   4SG jr5S3\\$   4SH jr6SI\4SJ jr7SK\SL\\\\4      S9\\8\94   4SM jr:SK\SL\\\\4      S9\\8\94   4SN jr;\<SK\SL\\=   SO\ SP   S9\94SQ j5       r>\< GS#SK\SL\\=   SO\ S   S9\84SR jj5       r>\< GS#SK\SL\\=   SO\\ SP   \ S   4   S9\\9\84   4SS jj5       r> GS#SK\SL\\=   SO\	4ST jjr>SU\9SL\\\\4      SV\S9\94SW jr?SK\SL\\\\4      S9\\8\94   4SX jr@ GS$SK\SY\SZ\\   S9S4S[ jjrA GS$SY\SZ\\   S9S4S\ jjrB GS%SI\SY\S]\\   S9\C4S^ jjrD GS%SI\SY\S]\\   S9S4S_ jjrESI\SY\4S` jrFSY\Sa\S9\\\\4      4Sb jrGSY\Sa\S9\\\\4      4Sc jrHSY\Sa\S9\\\\4      4Sd jrISe\\   SL\\\\\4      \\\\\4         4   4Sf jrJSK\SL\\\=      4Sg jrK\<SK\SL\\\\4      SO\ SP   S9\94Sh j5       rL\< GS#SK\SL\\\\4      SO\ S   S9\84Si jj5       rL GS#SK\SL\\\\4      SO\	4Sj jjrL\< GS#SK\SL\\=   Sk\SO\ S   S9\84
Sl jj5       rM\<SK\SL\\=   Sk\SO\ SP   S9\94
Sm j5       rM GS#SK\SL\\=   Sk\4Sn jjrMSK\Sk\So\Sp\N\Sq4   SY\\\4   4
Sr jrOSK\S9\	4Ss jrPSK\SL\\=   SY\\\4   4St jrQSu\SK\4Sv jrRSu\SK\4Sw jrSSu\SK\4Sx jrTSu\SK\4Sy jrUSz\VSK\4S{ jrWSz\VSK\4S| jrXSK\S}\S~\4S jrYSK\4S jrZSK\4S jr[   GS&SK\Su\S\\	   S\\	   S\\	   4
S jjr\   GS&SK\Su\S\\	   S\\	   S\\	   4
S jjr]SK\Su\4S jr^   GS&S\SK\S\\	   S\\	   S\\	   4
S jjr_S\SK\4S jr`So\4S jraSK\S\4S jrbSK\So\4S jrcSY\\\4   SK\S\S9\\\4   4S jrdSK\S\4S jreSK\So\4S jrf GS#SK\S}\\\4   S\\	   S9\g4S jjrhS}\\\4   SK\4S jri GS'SK\S}\\\4   S\\	   S9\g4S jjrjS}\\\4   SK\4S jrkSK\S9\l4S jrmSK\S9\l4S jrnSK\S9\o4S jrpSK\S9\o4S jrq GS%SK\\   S9\o4S jjrrSK\4S jrs GS%So\S\\   4S jjrt GS(So\S\ S   4S jjru GS%So\S\\   4S jjrvSo\4S jrw  GS)So\S\\   S\\x   4S jjryS\zS\\	   S$\\   S%\\   S&\\   S\\   Sp\SY\4S jr{\|R                  " 5       S 5       r~    GS*SY\S\\   S$\\   S%\\   S&\\   4
S jjr\|R                  " 5       S 5       rSo\4S jr GS%SY\S\\   4S jjr     GS+S\zS\\   S\\   S%\\   S&\\   S\\   4S jjrS r GS%S$\\\\\   4      S\\   S9\\\      4S jjrS9\\   4S jr  GS)S\zS\S\S\\   S\\   S9\\\4   4S jjrS rS9\\   4S jrS9\	4S jrS\\   4S jrSY\S9\ S   4S jrSY\S\zS9\4S jrS\S\\   S9\4S jrS9\	4S jrSK\S\8SY\S9\	4S jrSK\S\\   4S jrSK\S\\   S9\N\\   \\   4   4S jrSI\4S jr GS%SI\S\\   S\\   4S jjr  GS)SK\S\\   SL\\\=      S\\   S\\   S\\   4S jjrS\S\4S jrS\S\S\S\S9\\C   4
S jrS\S9\	4S jrSI\C4S jrSI\CS9\	4S jrS\4S jrSI\CS9\C4S jrSI\CS\SK\4S jrSI\CS9\\C   4S jrS\S\S9S4S jr GS%SK\S\\   S9S4S jjrSI\CS9\\C   4S jrS\S9\\C   4S jrS\S9\\C   4S jrS\S9\\   4S jrS\S9\\C   4S jr\< GS%SI\S\SSS9\4S jj5       r\<SISS\S\S9\4S j5       r GS%SI\\   S\S\\   S9\4S jjrS\S9\\   4S jrS\S9\\   4S jrS\S\S9\\   4S jrS\S\S9\\   4S jrS\S9\\   4S jrS\S9\N\\   \\   4   4S jr\" \S9S\S9\\   4S j5       rS\S9\\\4   4S jr GS%S\S\\   S9\4S jjrS\S9S4S jrS\4S jr GS,S\\   S\	S9\\   4S jjrS\S9\	4S jrS\S\S9\\   4S jr GS%S\SK\S\\   S9\	4S jjr  GS)S\GS \\   S\\   S9\\   4GS jjrGS%S\\   S9\\   4GS jjr GS%SI\S\\   S9\\   4GS jjrGS\\   S9\	4GS jr GS%S\\   S9\\   4GS jjr GS)S\\   S\\   S9\\\      4GS jjr   GS-S\\   GS\\   S\\   S9\\\\   4   4GS	 jjrGS\S9\	4GS
 jrGS rGS rGS%GS jr GS%SK\S\SL\\\\4      S\\   4GS jjrSK\S9\\   4GS jrSK\S9\4GS jr    GS.SK\SL\\\\\4         S}\\\\4      GS\\	   S\\   S9\N\\\\4   4   4GS jjr    GS.SK\S\SL\\\\\4         S}\\\\4      GS\\	   S\\   S9\\\   \4   4GS jjr   GS/SK\S\SL\\\\\4         S}\\\\4      GS\\	   4
GS jjr   GS/SK\S\SL\\\\\4         S}\\\\4      GS\\	   S9\\   4GS jjr    GS.SK\SL\\\\\4         S}\\\\4      GS\\	   S\\   4
GS jjrS\\   GS\\   S9\\   4GS jr GS%S\\   4GS jjr GS%GS\zS\\   4GS jjrGS\z4GS jrGS rGS\4GS jrGS  rGS! rGS"rg(0  Router   model_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggeroptional_callbacksINFOr   simple-shuffle
model_listassistants_configsearch_tools	redis_url
redis_host
redis_portredis_passwordcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGr   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)r   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsignore_invalid_deploymentsreturnc+                    UU l         U*U l        UU l        UU l        UU l        SSKJn+  U+" 5       U l        S[        l	        U R                   SL aU  US:X  a%  [        R                  " [        R                  5        O*US:X  a$  [        R                  " [        R                  5        U)=(       d
    [        5       U l        X l        U=(       d    / U l        / U l        0 U l        Sn,Sn-0 n.Xl        Uc  UbP  UbM  Sn,Ub  UU.S	'   Ub  UU.S
'   Ub  [-        U5      U.S'   Ub  UU.S'   U.R/                  U	5        U R1                  U.5      n-U(       a8  [        R2                  c!  [        R4                  " S SU,0U.D6[        l        Xl        [9        U-[;        5       S9U l        [=        UU-S9U l        Xl         SU l!        UU l"        / U l#        [I        5       U l%        0 U l&        0 U l'        U=(       d    0 U l(        0 U l)        0 U l*        UbL  U RW                  U5        U RX                  U l-        U H#  n/SU/S   ;   d  M  SU R(                  U/S   S   '   M%     O/ U l,        U b  U U l.        O[        R\                  U l.        U"=(       d    [^        U l0        [c        U R2                  U R`                  S9U l2        U#U l3        [;        5       U l4        Ub  Xl5        O<[        Rj                  b  [        Rj                  U l5        O[l        Rn                  U l5        Ub  Xl8        O<[        Rp                  b  [        Rp                  U l8        O[        Rr                  U l8        U=(       d    [        Rt                  U l;        UU l<        UU l=        U$U l>        U=(       d    [        R~                  n0U R                  U0S9  U0U l?        Uc  [        R                  bN  U=(       d    [        R                  n0U R~                  b  U R~                  R                  SU005        O
SU00/U l?        U=(       d    [        R                  U lC        U=(       d    [        R                  n1U R                  U1S9  U1U lD        [        [        5      U lG        [        [        5      U lH        [        [        5      U lI        / U lJ        U=(       d    0 n[        R                  " UU S9U lL        UU lM        U R                  R                  SU5        U R                  R                  SS5        U R                  R                  S0 5      R/                  SU
05        0 U lO         U R                  U$U&S9  SU lQ        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        O)[        R                  R                  U R                  5        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        lX        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        l[        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        l^        U&U la        U'U lb        SU lc        [        R                  " XR                  S9(       a  U%b  U%R                  S5        OS/n%SU lf        Ub~  [        U[        5      (       a  [        S 0 UD6U lf        O[        U[        5      (       a  UU lf        [        R                  " SR                  U R                  R                  SS95      5        UU ll        SU lm        U!b~  [        U![        5      (       a  [        S 0 U!D6U lm        O[        U![        5      (       a  U!U lm        [        R                  " SR                  U R                  R                  SS95      5        U(U lo        U%b  U R                  U%5        U R                  b  U R                  5         U R                  5         U R                  5         U R                  5         g)!ah  
Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

Args:
    model_list (Optional[list]): List of models to be used. Defaults to None.
    redis_url (Optional[str]): URL of the Redis server. Defaults to None.
    redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
    redis_port (Optional[int]): Port of the Redis server. Defaults to None.
    redis_password (Optional[str]): Password of the Redis server. Defaults to None.
    cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
    cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
    caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
    client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
    polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
    default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
    num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
    timeout (Optional[float]): Timeout for requests. Defaults to None.
    default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
    set_verbose (bool): Flag to set verbose mode. Defaults to False.
    debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
    fallbacks (List): List of fallback options. Defaults to [].
    context_window_fallbacks (List): List of context window fallback options. Defaults to [].
    enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
    model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
    retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
    allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
    cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
    routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
    routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
    alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
    provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
    ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
Returns:
    Router: An instance of the litellm.Router class.

Example Usage:
```python
from litellm import Router
model_list = [
{
    "model_name": "azure-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "azure/<your-deployment-name-1>",
        "api_key": <your-api-key>,
        "api_version": <your-api-version>,
        "api_base": <your-api-base>
    },
},
{
    "model_name": "azure-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "azure/<your-deployment-name-2>",
        "api_key": <your-api-key>,
        "api_version": <your-api-version>,
        "api_base": <your-api-base>
    },
},
{
    "model_name": "openai-gpt-3.5-turbo", # model alias
    "litellm_params": { # params for litellm completion/embedding call
        "model": "gpt-3.5-turbo",
        "api_key": <your-api-key>,
    },
]

router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
```
r   )ServiceLoggingTr   r   localNredisurlhostportpasswordtype)redis_cachein_memory_cache)r   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr   max_retriesmetadatar   r   r   )r   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0mr{   )ur   r   r   r   r   litellm._service_loggerr   service_logger_objlitellmsuppress_debug_infor   setLevelloggingr   r   rZ   r   r   r   deployment_namesdeployment_latency_mapr   strupdate_create_redis_cacher   Cacher   r   r   rF   	schedulerr   default_deploymentr   provider_default_deployment_idsrs   pattern_routerteam_pattern_routersauto_routersr    model_id_to_deployment_index_map model_name_to_deployment_indicesset_model_listr   healthy_deploymentsr   r6   r   r5   cooldown_cacher   failed_callsr   openaiDEFAULT_MAX_RETRIESr   ROUTER_MAX_FALLBACKSrequest_timeoutr   r   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsChatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groups
isinstance_async_success_callbacklistlogging_callback_manager"add_litellm_async_success_callbackdeployment_callback_on_successsuccess_callbackadd_litellm_success_callback#sync_deployment_callback_on_success_async_failure_callback"add_litellm_async_failure_callback$async_deployment_callback_on_failurefailure_callbackadd_litellm_failure_callbackdeployment_callback_on_failurer   r   router_budget_loggerr%   !should_init_router_budget_limiterr   dictrX   infoformat
model_dumpr   r   rN   r   add_optional_pre_call_checks_initialize_alertinginitialize_assistants_endpointinitialize_router_endpointsapply_default_settings)2selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbackss2                                                     r   __init__Router.__init__   s   l '*D'&&<#$8!:2@2B&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3(.B 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z( -22<@K}}$ ' N: N N#2 #]_


 #-;
 !1"&-J*:<,02  	! 68 # 	
 AC- GI-!
+-1__D$a 011PQD//2B0CG0LM  
  O $!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*  ,&22D%99D$!."".!(!6!6D!(!=!=D9'"9"9,& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<<,,OO33 ,,OO33 g..55,,II88 )-(P(P'QG$g55t<<,,OO99
 99/G+ g..55,,II33 )-(K(K'LG$%:"&<#DH!AA!:U:U
 (3(//0HI,D+E(37#,--$/$?,$?!L+66$0!!&&DKK%%00d0C % 	% CG!+.55,>,VAU,V)02DEE,@)!&&LSS--88d8K :I#/--.FG+%%'++-((*##%r   c                 *    / nU R                  U5        g)z+
Apply the default settings to the router.
N)r  )r  default_pre_call_checkss     r   r  Router.apply_default_settings{  s    
 :<))*ABr   c                 $   [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R
                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        [         R                  R                  [         R                  U 5        U R                  bA  U R                   H0  n[         R                  R                  [         R                  USS9  M2     gg)z
Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
For now, unhook router's callbacks from all lists
NF)require_self)r   r  #remove_callback_from_list_by_objectr  r  r
  r  input_callbackservice_callback	callbacksr   )r  callbacks     r   discardRouter.discard  s3   
 	((LL++T	
 	((LL$$d	
 	((LL++T	
 	((LL$$d	
 	((LL""D	
 	((LL$$d	
 	((LLt	

 "". 3300TT%%xe U  4 /r   r  c                 Z    U R                  S5      (       a  [        S0 U D6$ [        S0 U D6$ )zQ
Initializes either a RedisCache or RedisClusterCache based on the cache_config.
startup_nodesr{   )getr   r   )r  s    r   r   Router._create_redis_cache  s1     O,,$4|44---r   r   c                 T    U R                   R                  c  XR                   l        gg)z
Update the redis cache for the router, if none set.

Allows proxy user to just do
```yaml
litellm_settings:
    cache: true
```
and caching to just work.
N)r   r   )r  r   s     r   _update_redis_cacheRouter._update_redis_cache  s#     ::!!)%*JJ" *r   c                    [         R                  " SU 35        U[        R                  R                  :X  d  U[        R                  :X  a  [        U R                  S9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        OU R                  /[        l
        [        [        R                  [        5      (       a*  [        R                  R                  U R                  5        g g U[        R                   R                  :X  d  U[        R                   :X  ag  [#        U R                  US9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R$                  5        g g U[        R&                  R                  :X  d  U[        R&                  :X  ag  [)        U R                  US9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R*                  5        g g U[        R,                  R                  :X  d  U[        R,                  :X  ag  [/        U R                  US9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R0                  5        g g U[        R2                  R                  :X  d  U[        R2                  :X  ag  [5        U R                  0 S9U l        [        [        R                  [        5      (       a*  [        R                  R                  U R6                  5        g g g )NzRouting strategy: )router_cache)r7  routing_args)r   r  r^   
LEAST_BUSYvaluer&   r   r   r  r   r)  r  r   r+  r  add_litellm_callbackUSAGE_BASED_ROUTINGr)   r   USAGE_BASED_ROUTING_V2r*   lowesttpm_logger_v2LATENCY_BASEDr(   lowestlatency_logger
COST_BASEDr'   lowestcost_logger)r  r   r   s      r   r   Router.routing_strategy_init  s    	""%78H7I#JK : : @ @@?#=#==$;$TD!'00$77&&--d.C.CD*.*?*?)@&'++T2200EEdF[F[\ 3  C C I II?#F#FF$;!ZZ2%D! '++T2200EEdF[F[\ 3  F F L LL?#I#II'A!ZZ2(D$ '++T2200EEdF^F^_ 3  = = C CC?#@#@@(C!ZZ2)D% '++T2200EEdF_F_` 3  : : @ @@?#=#==%=!ZZ&D" '++T2200EEdF\F\] 3 r   c                 D   U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R
                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l        U R                  [        R                  5      U l	        g N)
factory_functionr   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr  s    r   r  %Router.initialize_assistants_endpoint  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEr   c                 |   U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R
                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l	        U R                  [        R                  S	S9U l
        U R                  [        R                  S
S9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                  SS9U l        U R                  [        R                   SS9U l        U R                  [        R"                  SS9U l        U R                  [        R$                  SS9U l        U R                  [        R&                  SS9U l        U R                  [        R(                  SS9U l        U R                  [        R*                  SS9U l        U R                  [        R,                  SS9U l        g)z+Helper to initialize core router endpoints.
moderation	call_typeanthropic_messagesagenerate_contentaadapter_generate_content
aresponsesafile_deleteafile_content	responsesaget_responsesacancel_responsesadelete_responsesalist_input_items
_arealtimeacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_filesaimage_editallm_passthrough_routeN)rF  r   amoderationrU  aanthropic_messagesrV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r`  ra  rb  rc  rd  
afile_listrf  rg  rO  s    r   _initialize_core_endpoints!Router._initialize_core_endpoints  s   00< 1 
 $(#8#8&&2F $9 $
  "&!6!6%%1D "7 "
 *.)>)>--9T *? *
& //, 0 
 !11  N 2 
 "22!!_ 3 
 ..w/@/@K.X"33"".> 4 
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 //, 0 
 (,'<'<++7P (= (
$ (,'<'<++7P (= (
$ '+&;&;**6N '< '
# *.)>)>--9T *? *
& //- 0 
  00= 1 
 '+&;&;**6N '< '
#r   c                    SSK JnJnJnJn  U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l	        SSK
JnJnJnJnJn	Jn
JnJnJnJnJnJn  U R                  US	S9U l        U R                  US
S9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  USS9U l        U R                  U	SS9U l        U R                  USS9U l        U R                  U
SS9U l        U R                  USS9U l         SSK!J"nJ#nJ$nJ%n  U R                  USS9U l"        U R                  USS9U l$        U R                  USS9U l#        U R                  USS9U l%        SSK&J'nJ(n  U R                  USS9U l'        U R                  USS9U l(        SSK)JnJn  U R                  USS9U l        U R                  USS9U l        SS K*J+nJ,nJ-nJ.nJ/nJ0nJ1nJ2nJ3nJ4n   U R                  US!S9U l,        U R                  US"S9U l1        U R                  US#S9U l-        U R                  US$S9U l2        U R                  US%S9U l/        U R                  U S&S9U l4        U R                  US'S9U l+        U R                  US(S9U l0        U R                  US)S9U l.        U R                  US*S9U l3        SS+K5J6n!J7n"J8n#J9n$J:n%J;n&J<n'J=n(  U R                  U!S,S9U l6        U R                  U%S-S9U l:        U R                  U#S.S9U l8        U R                  U'S/S9U l<        U R                  U$S0S9U l9        U R                  U(S1S9U l=        U R                  U"S2S9U l7        U R                  U&S3S9U l;        g4)5z`Helper to initialize specialized router endpoints (vector store, OCR, search, video, container).r   )acreateasearchcreatesearchavector_store_searchrS  avector_store_createvector_store_searchvector_store_create)rn  alist	aretrievearetrieve_contentaupdateadeleterp  r  retrieveretrieve_contentr   deleteavector_store_file_createvector_store_file_createavector_store_file_listvector_store_file_listavector_store_file_retrievevector_store_file_retrieveavector_store_file_contentvector_store_file_contentavector_store_file_updatevector_store_file_updateavector_store_file_deletevector_store_file_delete)rV  agenerate_content_streamgenerate_contentgenerate_content_streamrV  r  r  r  )aocrocrr  r  ro  rq  ro  rq  )
avideo_contentavideo_generationavideo_listavideo_remixavideo_statusvideo_contentvideo_generation
video_listvideo_remixvideo_statusr  r  r  r  r  r  r  r  r  r  )acreate_containeradelete_containeralist_containersaretrieve_containercreate_containerdelete_containerlist_containersretrieve_containerr  r  r  r  r  r  r  r  N)>litellm.vector_stores.mainrn  ro  rp  rq  rF  rr  rs  rt  ru  litellm.vector_store_files.mainrv  rw  rx  ry  rz  r  r{  r|  r   r}  r~  r  r  r  r  r  r  r  r  r  r  r  litellm.google_genairV  r  r  r  litellm.ocrr  r  litellm.searchlitellm.videosr  r  r  r  r  r  r  r  r  r  litellm.containersr  r  r  r  r  r  r  r  ))r  rn  ro  rp  rq  avector_store_file_create_fnavector_store_file_list_fnavector_store_file_retrieve_fnavector_store_file_content_fnavector_store_file_update_fnavector_store_file_delete_fnvector_store_file_create_fnvector_store_file_list_fnvector_store_file_retrieve_fnvector_store_file_content_fnvector_store_file_update_fnvector_store_file_delete_fnrV  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s)                                            r   !_initialize_specialized_endpoints(Router._initialize_specialized_endpointsC  sN   OO$($9$95 %: %
! %)$9$95 %: %
! $(#8#83 $9 $
  $(#8#83 $9 $
 	
 	
 	
 	
 *.)>)>(4O *? *
& )-(=(='3M )> )
% (,'<'<&2K (= (
$ '+&;&;%1I '< '
# ,0+@+@*6S ,A ,
( +/*?*?)5Q +@ +
' +/*?*?)5Q +@ +
' *.)>)>(4O *? *
& *.)>)>(4O *? *
& )-(=(='3M )> )
% *.)>)>(4O *? *
& )-(=(='3M )> )
%	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 )-(=(=$0J )> )
% (,'<'<#/H (= (
$ 	*))$&)A	(((>2,,W	,J++Fh+G	
 	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
  000V//
l/S!22_ 3 
 !11N 2 
 #33&6 4 
 "22_ 3 
 !11N 2 
  000V		
 		
 		
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 "&!6!6)< "7 "
 !% 5 5(: !6 !
r   c                 D    U R                  5         U R                  5         g rE  )rk  r  rO  s    r   r  "Router.initialize_router_endpoints  s    '')..0r   r   c           	          Uc  gU HQ  n[        U[        5      (       d  [        SU S35      e[        U5      S:w  d  M8  [        SU S[        U5       S35      e   g)z#
Validate the fallbacks parameter.
NzItem 'z' is not a dictionary.rr   zDictionary 'z%' must have exactly one key, but has z keys.)r  r  
ValueErrorlen)r  r   fallback_dicts      r   r   Router.validate_fallbacks  so     !+MmT22 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 	 ,r   c                 |   Ub  U H  nS nUS:X  a  [        U R                  S9nO@US:X  a*  [        U R                  U R                  U R                  S9nOUS:X  a
  [        5       nUc  Md  U R                  c  / U l        U R                  R                  U5        [        R                  R                  U5        M     g g )Nprompt_caching)r   r   )
dual_cacher   r   responses_api_deployment_check)rA   r   r%   r   r   rB   r   r   r   r  r;  )r  r   pre_call_check	_callbacks       r   r  #Router.add_optional_pre_call_checks  s     $/":48	!%55 <4:: NI#'?? 4#'::/3/J/J#'??!I
 $'GG ; =I(..624/++229=44II)T! #; 0r   
deploymentc                 <    [         R                  " U5      nUS   n[        R                  (       a   [	        SSS9nUR                  U5      US'   U$ SU;   a  US   SS S-   US'   U$ ! [         a)  n[        R                  " S[        U5       35        UeSnAff = f)	z
returns a copy of the deployment with the api key masked

Only returns 2 characters of the api key and masks the rest with * (10 *).
r      r   )visible_prefixvisible_suffixapi_keyNz
**********z+Error occurred while printing deployment - )
copydeepcopyr   redact_user_api_key_infor$   	mask_dict	Exceptionr   debugr   )r  r  _deployment_copyr   maskeres         r   print_deploymentRouter.print_deployment  s    	#}}Z8#34D#EN//,AaP5;5E5En5U !12 $# n,,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   AA( A( (
B2$BBr   messagesc                      [         R                  " SU S35        XS'   X#S'   U R                  US'   U R                  XS9  U R                  " S0 UD6nU$ ! [
         a  nUeSnAff = f)	z
Example usage:
response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
zrouter.completion(model=z,..)r   r  original_functionr   kwargsNr{   )r   r  _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr  )r  r   r  r  responser  s         r   
completionRouter.completion0  s}    
	!''*B5'(NO#7O!):*.*:*:F&'00u0L33=f=HO 	G	s   AA 
A' A""A'c           	         S n U R                  UUUR                  SS 5      US9nU R                  XSS9  US   nUS   nU R                  XSS9nUR	                  SS 5      nUb  Ub  XR
                  :w  a  S n	OUn	U R                  U5      (       d  U R                  US9  [        R                  " S0 0 UEUU R                  U	S.EUED6n
[        R                  " S	U S
35        [        U
[        5      (       a-  U R                  XUS9nU(       a  [        R                   " SUSS9eU
$ ! ["         a-  n[        R                  " S	U S[%        U5       S35        UeS nAff = f)Nspecific_deploymentr   r  r  request_kwargsr  r  r   r   r  r  r  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r  r  Response output was blocked. messager   llm_provider)[31m Exception [0mr{   )get_available_deploymentpop_update_kwargs_with_deployment_get_clientr1  r  has_model_id routing_strategy_pre_call_checksr   r  r   r   r  r  rj   "_should_raise_content_policy_errorContentPolicyViolationErrorr  r   )r  r   r  r  
model_namer  datapotential_model_clientdynamic_api_keymodel_clientr  _should_raiser  s                r   r  Router._completionC  s    
>	66!$*JJ/Dd$K%	 7 J //:/U ./DgJ%)%5%5% &6 &" %jjD9O+*6#'E'EE#5 $$U++555L))  (#33*	
 H "&&+J<7NO
 (M22 $ G G6 !H ! !!== >#%'  O 	!&&+J<7KCPQF8SZ[ G		s   D"D' '
E1(EEstreamTc                    #    g 7frE  r{   r  r   r  r  r  s        r   acompletionRouter.acompletion  
      	   c                    #    g 7frE  r{   r  s        r   r  r    r  r  c                    #    g 7frE  r{   r  s        r   r  r    r  r  c                 0  #     XS'   X$S'   X4S'   U R                   US'   U R                  XS9  UR                  S5      =(       d    U R                  n[        R
                  " 5       nU R                  U5      nU(       a  U R                  UUUS9I S h  vN $ Ub0  [        U[        5      (       a  U R                  " S0 UD6I S h  vN nOU R                  " S0 UD6I S h  vN n[        R
                  " 5       n	X-
  n
[        R                  " U R                  R                  [         R"                  U
SUU	[%        U5      S	95        U$  N N Nl! [&         a:  n[        R                  " [)        U U[*        R,                  " 5       US
95        UeS nAff = f7f)Nr   r  r  r  r  priority)r   r  r  r  servicedurationrT  
start_timeend_timeparent_otel_spanlitellm_router_instancer  error_traceback_stroriginal_exceptionr{   )_acompletionr  r1  r   timeperf_counter_is_prompt_management_model_prompt_management_factoryr  r   schedule_acompletionasync_function_with_fallbacksasynciocreate_taskr   async_service_success_hookr`   ROUTERr   r  r@   	traceback
format_exc)r  r   r  r  r  request_priorityr  r  r  r  	_durationr  s               r   r  r    s    ,	#7O!):%8*.*;*;F&'00u0L%zz*5N9N9N**,J*.*J*J5*Q'*!<<%! =   
  +
;KS0Q0Q!%!:!:!DV!DD!%!C!C!Mf!MM((*H -I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE E	E F-E EE  E!A'E F	E E E 
F5FFFmodel_responseinitial_kwargsc                 r   ^ ^^^^#    SSK Jm   " U4S jS[        5      nUUUUU 4S jnU" U" 5       5      $ 7f)zo
Helper to iterate over a streaming response.

Catches errors for fallbacks using the router's fallback system
r   )MidStreamFallbackErrorc                   @   >^  \ rS rSrS\4U U4S jjrS rS rSrU =r	$ )ERouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapperi  async_generatorc                 n   > [         TU ]  UTR                  TR                  TR                  S9  Xl        g )N)completion_streamr   custom_llm_providerlogging_obj)superr!  r   r/  r0  _async_generator)r  r,  	__class__r&  s     r   r!  NRouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__init__  s;     &5(..(6(J(J . : :	 !  )8%r   c                     U $ rE  r{   rO  s    r   	__aiter__ORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__aiter__  s    r   c                 R   #    U R                   R                  5       I S h  vN $  N7frE  )r2  	__anext__rO  s    r   r9  ORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__anext__  s      !22<<>>>>s   '%')r2  )
r|   r}   r~   r   r   r!  r6  r9  r   __classcell__)r3  r&  s   @r   FallbackStreamWrapperr+    s#    8 8 8? ?r   r<  c                 |  >#     T  S h  vN n U 7v   M   N
 g ! T Ga  nSSK Jn  U" TR                  S9n[        [        [
           [        USS 5      5      n [        [        TR                  S5      5      nTR                  STR                  5      nTR                  STR                  5      nTR                  STR                  5      nTR                  TS	'   TS
SS.SUR                  SS./-   TS'   TR                  UTS9  TR                  USUUUUSTS9I S h  vN  n	[!        U	S5      (       a  U	  S h  vN  n
U
(       a  [#        U
[$        5      (       a  [!        U
S5      (       a  SSKJn  [        [        [
           [        U
SS 5      5      nUb  U/nO/ nUb/  [!        US5      (       a  UR*                  b  UR-                  U5        UR/                  US9n[1        U
SU5        U
7v   M  
  S nAg S 7v    S nAg ! [2         a   n[4        R6                  " SU 35        UeS nAff = fS nAff = f7f)Nr   )stream_chunk_builder)chunksusager   r   r   r   r  systema  You are a helpful assistant. You are given a message and you need to respond to it. You are also given a generated content. You need to respond to the message in continuation of the generated content. Do not repeat the same content. Your response should be in continuation of this text: rolecontent	assistantT)rC  rD  prefixr  r  Fr{   )r  disable_fallbacksr   r   r   model_groupargsr  r6  )BaseTokenUsageProcessor)usage_objectszFallback also failed: )litellm.mainr>  r?  r   r   rg   getattrr   r1  r   r   r   r  generated_contentr  *async_function_with_fallbacks_common_utilshasattrr  re   litellm.cost_calculatorrJ  r@  r   combine_usage_objectssetattrr  r   error)itemr  r>  complete_response_objectcomplete_response_object_usagerH  r   r   r   fallback_responsefallback_itemrJ  r@  rK  combined_usagefallback_errorr)  r'  r  r&  r  s                   r   stream_with_fallbacksERouter._acompletion_streaming_iterator.<locals>.stream_with_fallbacks  s    a)"0 $J.) ^)=+?)00,( 26UO4gtD2.T)"&sN,>,>w,G"HK0>0B0B#T^^1I @N?Q?Q2D4Q4Q@, @N?Q?Q2D4Q4Q@, ;?:K:KN#6719$, (I
 %0'(':':&*
= 
2N:. 88). 9  #MM.3&/5M5M(3!##1 N 	 	 	 & 0+>>3D  0  0- -$.}>Q$R$R$+M7$C$C!" )-$,UO$+M7D$I)" $)#45:GM46M %C$N(/0NPW(X(X(F(L(L(X$1$8$89W$X %<$Q$Q6C %R %& !/
 !(w O"//A 4EF #

  ))//00@A )()s^)s   H<   H<H98H4CH!D$"H:G7>E?G7B5H8H<=HH<
H1H,,H11H44H99H<)litellm.exceptionsr)  rh   )r  r&  r  r'  r<  r\  r)  s   ````  @r   _acompletion_streaming_iterator&Router._acompletion_streaming_iterator  s6      	>	?$7 	?"b	) b	)H %%:%<==s   07c                 8  #    Sn0 n UR                  5       nXS'   [        U5      n[        R                  " 5       nU R	                  UUUR                  SS5      US9I Sh  vN n	U	n[        R                  " 5       n
X-
  n[        R                  " U R                  R                  [        R                  USUU
[        U5      S95        U R                  XS9  U R                  XS9  U	S	   nUS   nU R                  U	US9nU R                  U==   S
-  ss'   0 UEUU R                   US.EUEn["        R$                  " S0 UD6nUR'                  SS5      nU R)                  U	USS9nUb_  [+        U[        R,                  5      (       a@  U ISh  vN    U R/                  U	UUS9I Sh  vN   UI Sh  vN nSSS5      ISh  vN   O#U R/                  U	UUS9I Sh  vN   UI Sh  vN n[+        W[0        5      (       a.  U R3                  UUUS9nU(       a  ["        R4                  " SUSS9eU R6                  U==   S
-  ss'   [8        R:                  " SU S35        U R                  U	UUS9  [+        U[<        5      (       a  U R?                  UUUS9I Sh  vN $ U$  GNA GN N N N! , ISh  vN  (       d  f       N= f N N N0! ["        R@                   af  nUR'                  S	0 5      R'                  SS5      nUR'                  S	0 5      R'                  SS5      nU=RB                  SU SU 3-  sl!        UeSnAf[D         aG  n[8        R:                  " SU S[G        U5       S35        Ub  U RH                  U==   S
-  ss'   UeSnAff = f7f)z
- Get an available deployment
- call it with a semaphore over the call
- semaphore specific to it's rpm
- in the semaphore,  make a check against it's local rpm before running
Nr   r  r  async_get_available_deploymentr  r  r  r  r   rr   r  litellm_logging_objmax_parallel_requestsr  r  client_type)r  r0  r  r  r  r  r  zlitellm.acompletion(model=r  )r  r  r  )r&  r  r'  r   r   z$

Deployment Info: request_timeout: z

timeout: r  r  r{   )%r  r   r  r  rb  r  r  r  r   r   r`   r!  _track_deployment_metricsr  _get_async_openai_model_clientr   r   r   r  r1  r  r  	Semaphore&async_routing_strategy_pre_call_checksrj   r  r  r   r   r  rh   r_  Timeoutr  r  r   r   )r  r   r  r  r  _timeout_debug_deployment_dict#input_kwargs_for_streaming_fallbackr  r  r  r  r%  r  r  input_kwargs	_responser0  rpm_semaphorer  r   r   deployment_request_timeout_paramdeployment_timeout_params                          r   r  Router._acompletionW  sQ     
 	'D	28++-/;@8@H**,J#BB!$*JJ/Dd$K%	  C   J .8*((*H -I''BB(//&>)%%Fv%N C 	 **% +  //:/U./DgJ>>% ? L Z(A-($//&	
 L  ++;l;I4:JJ%t5K !,,%3 - M
 (Zw00. . )= EE#-$/)9 F   
 &/H )== AA) +%5 B    "+? (M22 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  ($788!AA#+%#F B    O]x )

  / )=== +4  	/M/Q/Q "0c#T* - (F'I'I "(c)T" % IIABbAccn  pH  oI  J  JIG 	!&&,ZL8LSQRVHT[\ %
+q0+G	s   NAK !J""D.K J%K J.*J(+	J.4J*5J.9K J,K K	K (K
)B2K KK N K !N"K %K (J.*J.,K .K4J75KK 
K K N"A!MNANNNr  metadata_variable_namec                    UR                  SU R                  5      US'   UR                  S[        [        R
                  " 5       5      5        SnU R                  US9(       a  UnUR                  U0 5      R                  XS.5        g)zE
Adds/updates to kwargs:
- num_retries
- litellm_trace_id
- metadata
r   litellm_trace_idNr   )rH  r   )r1  r   r   r   r   uuid4_get_model_from_aliasr   )r  r   r  ru  r   s        r   r  &Router._update_kwargs_before_fallbacks  sz     !'

=$:J:J K},c$**,.?@+/%%E%2 %0"5<<!J	
r   c                    U R                   R                  5       nUR                  S0 5      =(       d    0 nUR                  5        H  u  pVUc  M
  UR	                  XV5        M     UR	                  U0 5      R                  U5        g)z
Adds default litellm params to kwargs, if set.

Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
r   N)r   r  r  itemsr   r   )r  r  ru  defaultsmetadata_defaultskeyr:  s          r   *_update_kwargs_with_default_litellm_params1Router._update_kwargs_with_default_litellm_params  sx     ..335$LLR8>B #..*JC}c) + 	0"5<<=NOr   function_namec                 h   UR                  S0 5      R                  5       nUS   R                  5       n[        XRS9n[        US9nUR                  U0 5      R                  S5      nU R	                  XS9n	UR                  S5      n
XS'   XS'   [        U[        S0 UD6US	9nU R                  US
9  U$ )z
Handle clientside credential

model_infor   )r   r  r  rH  )rH  r   idoriginal_model_idr  r   r  r  r{   )r1  r  r3   r/   _generate_model_idrR   rT   upsert_deployment)r  r  r  r  r  r   dynamic_litellm_paramsru  rH  	_model_idr  deployment_pydantic_objs               r   _handle_clientside_credential$Router._handle_clientside_credential  s      ^^L"5::<
#$45::<!;)"
 "D'"
 jj!7<@@O++# , 
	 'NN40$4*;&'",")C,BC!#

 	. 	 	
 '&r   c                    UR                  S0 5      R                  5       nUS   S   nUS   R                  S5      nUS   n[        US9(       aV  U R                  XUS9nUR                  R                  5       nUR                  R                  nUR                  R                  n[        US9n	UR                  U	0 5      R                  UUUUS	.5        XBS'   U R                  X!S   S
9US'   U R                  X)S9  g)z
2 jobs:
- Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
- Adds default litellm params to kwargs, if set.
r  r   r   api_baser  )r  r  r  r  r  )r  r  r  deployment_model_namer  r  r   )r  ru  N)r1  r  r4   r  r  r  r   r   r  r/   r   r   _get_timeoutr  )
r  r  r  r  r  deployment_litellm_model_namedeployment_api_baser  r  ru  s
             r   r  %Router._update_kwargs_with_deployment4  s0     ^^L"5::<
(23C(DW(M%()9:>>zJ *< 8#6:&*&H&H%M 'I '# 1;;FFHJ,C,R,R,X,X)"9"H"H"Q"Q!C'"
 	0"5<<;(/)>		
  *| --+; < . 
y 	77 	8 	
r   c                     U R                  XSS9nUR                  SS5      nUb  Ub  XCR                  :w  a  SnU$ UnU$ )aY  
Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

The same OpenAI client is re-used to optimize latency / performance in production

If dynamic api key is provided:
    Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
asyncrf  r  N)r  r1  r  )r  r  r  r  r  r  s         r   ri  %Router._get_async_openai_model_clienta  sc     "&!1!1!g "2 "

 !**Y5'&2#A#AAL  2Lr   r  c                     UR                  SS5      =(       dH    UR                  SS5      =(       d/    U R                  =(       d    U R                  R                  SS5      $ )z=Helper to get stream timeout from kwargs or deployment paramsr   N)r1  r   r   )r  r  r  s      r   _get_stream_timeoutRouter._get_stream_timeout{  s_    
 JJ'. Gxx $G ""	G
 **../?F	
r   c                 ,   UR                  SS5      =(       dz    UR                  SS5      =(       da    UR                  SS5      =(       dH    UR                  SS5      =(       d/    U R                  =(       d    U R                  R                  SS5      nU$ )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr   )r1  r   r   r  r  r  r   s       r   _get_non_stream_timeoutRouter._get_non_stream_timeout  s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 r   c                 z    SnUR                  SS5      (       a  U R                  XS9nUc  U R                  XS9nU$ )z6Helper to get timeout from kwargs or deployment paramsNr  Fr  )r1  r  r  r  s       r   r  Router._get_timeout  sO    /3::h&&..f.HG?22 3 G r   modelsc                 Z  ^ #    S[         S[        [           4U 4S jjnS[         S[        [           S[        4U 4S jjn[	        U[
        5      (       aX  [        S U 5       5      (       aA  / nU H  nUR                  U" SXrS.UD65        M     [        R                  " U6 I Sh  vN nU$ [	        U[
        5      (       a  [        S	 U 5       5      (       a  / n[        U5       H(  u  pU H  nUR                  U" SXyU
S
.UD65        M     M*     [        R                  " U6 I Sh  vN n[        [        U5      5       Vs/ s H  n/ PM     nnU HG  n[	        U[        5      (       a  XS      R                  US   5        M3  US   R                  U5        MI     U$ gg N N{s  snf 7f)av  
Async Batch Completion. Used for 2 scenarios:
1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

Example Request for 1 request to N models:
```
    response = await router.abatch_completion(
        models=["gpt-3.5-turbo", "groq-llama"],
        messages=[
            {"role": "user", "content": "is litellm becoming a better product ?"}
        ],
        max_tokens=15,
    )
```


Example Request for N requests to M models:
```
    response = await router.abatch_completion(
        models=["gpt-3.5-turbo", "groq-llama"],
        messages=[
            [{"role": "user", "content": "is litellm becoming a better product ?"}],
            [{"role": "user", "content": "who is this"}],
        ],
    )
```
r   r  c                 |   >#     TR                   " SXS.UD6I Sh  vN $  N! [         a  nUs SnA$ SnAff = f7fz[
Wrapper around self.async_completion that catches exceptions and returns them as a result
r   r  Nr{   r  r  r   r  r  r  r  s       r   _async_completion_no_exceptionsARouter.abatch_completion.<locals>._async_completion_no_exceptions  ?     !--WEWPVWWWW 0   <# !# <# 
949<9<idxc                    >#     TR                   " SXS.UD6I Sh  vN U4$  N! [         a  nXB4s SnA$ SnAff = f7fr  r  )r   r  r  r  r  r  s        r   *_async_completion_no_exceptions_return_idxLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idx  sL     **TTVTT T  vs0   ?% #% ?% 
<7<?<?c              3   B   #    U  H  n[        U[        5      v   M     g 7frE  )r  r  .0r  s     r   	<genexpr>+Router.abatch_completion.<locals>.<genexpr>  s     -T8ajD.A.A8   r  Nc              3   B   #    U  H  n[        U[        5      v   M     g 7frE  )r  r  r  s     r   r  r    s     /VX
1d0C0CXr  )r   r  r  rr   r   r{   )r   r   rG   r   r  r  allr   r  gather	enumerateranger  tuple)r  r  r  r  r  r  _tasksr   r  r  r  r[  _final_responsess   `             r   abatch_completionRouter.abatch_completion  s    H				"&'7"8				+,	 	& h%%#-T8-T*T*TF=gEg`fgh   %^^V44HO$''C/VX/V,V,VF )( 3#EMMB "'7FL $ !4 &nnf55I<A#h-<P/Q<Pq<PO/Q%h..#QK077D#A&--h7	 &
 #"# -W' 5 6/Qs8   B#F+&F"'BF+(F$)F+F&AF+$F+&F+c           	         ^ #    S[         S[        [           4U 4S jjn/ nU H  nUR                  U" SXS.UD65        M     [        R
                  " U6 I Sh  vN nU$  N7f)a  
Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

Use this for sending multiple requests to 1 model

Args:
    model (List[str]): model group
    messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
    **kwargs: additional kwargs
Usage:
    response = await self.abatch_completion_one_model_multiple_requests(
        model="gpt-3.5-turbo",
        messages=[
            [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
            [{"role": "user", "content": "hello good mornign"}],
        ]
    )
r   r  c                 |   >#     TR                   " SXS.UD6I Sh  vN $  N! [         a  nUs SnA$ SnAff = f7fr  r  r  s       r   r  ]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  r  r  r  Nr{   )r   r   rG   r   r  r  )r  r   r  r  r  r  message_requestr  s   `       r   -abatch_completion_one_model_multiple_requests4Router.abatch_completion_one_model_multiple_requests  ss     ,				"&'7"8		 'OMM/ =C  ( !00 1s   AA"A A"c                    #    g 7frE  r{   r  s        r   "abatch_completion_fastest_response)Router.abatch_completion_fastest_response0  r  r  c                    #    g 7frE  r{   r  s        r   r  r  8  r  r  c                   ^ ^#    UR                  S5       Vs/ s H  oUR                  5       PM     nnS[        S[        [        [        [        4      S[
        S[        S[        [        [        [        4   4
U 4S jjn/ mS[        R                  4U4S	 jjnU H3  n[        R                  " U" SXUS
.UD65      n	TR                  U	5        M5     T(       af  [        R                  " T[        R                   S9I Sh  vN u  n
mU
 H)  nU" U5      I Sh  vN nUc  M  SUR"                  S'   Us  $    T(       a  Mf  [        S5      es  snf  NO N97f)z
model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

Returns fastest response from list of model names. OpenAI-compatible endpoint.
,r   r  r  r  r   c                    >#     TR                   " SXUS.UD6I Sh  vN nU$  N! [        R                   a'    [        R                  " SR                  U 5      5        e [         a  nUs SnA$ SnAff = f7f)zV
Wrapper around self.acompletion that catches exceptions and returns them as a result
r   r  r  Nz4Received 'task.cancel'. Cancelling call w/ model={}.r{   )r  r  CancelledErrorr   r  r  r  )r   r  r  r  resultr  r  s         r   r  RRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptionsN  sy     	#//heW]haghh i)) %++JQQRWX  s>   A8& $& A8& AA5)A0*A5+A80A55A8taskc                   >#     U I S h  vN n[        U[        [        45      (       aC  [        R                  " S5        T H  nUR                  5         M     U TR                  U 5        $   TR                  U 5        g  Nv! [         a     $ f = f! [         a     N1f = f! [         a     g f = f!  TR                  U 5        f ! [         a     f f = f= f7f)Nz=Received successful response. Cancelling other LLM API calls.)	r  rj   rh   r   r  cancelremoveKeyErrorr  )r  r  tpending_taskss      r   check_responseARouter.abatch_completion_fastest_response.<locals>.check_responsea  s     #f}6I&JKK)//W +
 +!!((. L!((. $         !((. s   CB BAB B,C-B3 /B#  CB 
BCBC
B B3 B  B3 #
B0-C/B00C3C5CC
CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedr{   )splitstripr   r   r
   boolr   r   rj   rh   r  r  Taskr  r   waitFIRST_COMPLETED_hidden_params)r  r   r  r  r  r  r  r  r  r  donecompleted_taskr  r  s   `            @r   r  r  @  sR     &+[[%56%5'')%56		"&tCH~"6	@D	PS	="5y@A	" 	w|| 	, E&&/ 6EKD
   &  (/7+B+B) #D- #'-n==%QUF))*MN!M #'	 m *++C 7j# >s:   EE
CE?E EEE!E?EEr  c                    #    g 7frE  r{   r  r   r  r  r  r  s         r   r  Router.schedule_acompletion  r  r  c                    #    g 7frE  r{   r  s         r   r  r    r  r  c                   #    [        U5      n[        [        R                  " 5       5      n[	        UUSS9nU R
                  R                  US9I S h  vN   [        R                  " 5       U R                  -   n	[        R                  " 5       n
U R
                  R                  nSnX:  a  U R                  XS9I S h  vN u  pU R
                  R                  UR                  UR                  US9I S h  vN nU(       a  O:[        R                   " U5      I S h  vN   [        R                  " 5       n
X:  a  M  U(       a\   U R"                  " SXUS.UD6I S h  vN nUR$                  R'                  S0 5        UR$                  S   R)                  S	S
05        U$ [.        R0                  " SUSS9e GNd N N N Na! [*         a  n[-        USU5        UeS nAff = f7f)Nzgpt-3.5-turbor  
request_idr  requestFr   r  r  r  health_deploymentsr  additional_headers%x-litellm-request-prioritization-usedTr  %Request timed out while polling queuer   r  r{   )r   r   r   ry  rE   r   add_requestr  	monotonicr   r   _async_get_healthy_deploymentspollr  r  r  sleepr  r  r   r   r  rS  r   rl  )r  r   r  r  r  r  r  _request_idrU  r  	curr_timepoll_intervalmake_request_healthy_deploymentsr  rp  r  s                    r   r  r    s     =VD$**,'"&
 nn(((666 >>#dll2NN$	77",0,O,O -P - '#  "&!4!4????#7 "5 " L
 mmM222 NN,	 " "&"2"2 #6#EK# 	 ((334H"M(()=>EE<dC ! 
 //?% K 	7' 3
  :x0s   AG#F7A*G#>F:?8G#7F<8%G#F>G#>G#G G AG  G#:G#<G#>G# G 
G GG  G#r  rI  .c                   #    [        U5      n[        [        R                  " 5       5      n[	        UUUS9nU R
                  R                  US9I S h  vN   [        R                  " 5       U R                  -   n	[        R                  " 5       n
U R
                  R                  nSnX:  a  U R                  XS9I S h  vN u  pU R
                  R                  UR                  UR                  US9I S h  vN nU(       a  O:[        R                   " U5      I S h  vN   [        R                  " 5       n
X:  a  M  U(       an   U" U0 UD6I S h  vN n[#        UR$                  [&        5      (       a<  UR$                  R)                  S0 5        UR$                  S   R+                  SS05        U$ [0        R2                  " S
USS9e GNv GN N N N! [,         a  n[/        US	U5        UeS nAff = f7f)Nr  r  Fr  r  r  r  Tr  r  r   r  )r   r   r   ry  rE   r   r  r  r  r   r   r  r  r  r  r  r  r  r  r  r   r   r  rS  r   rl  )r  r   r  r  rI  r  r  r  rU  r  r   r  r  r  r  rp  r  s                    r   _schedule_factoryRouter._schedule_factory  s     =VD$**,'"
 nn(((666 >>#dll2NN$	77",0,O,O -P - '#  "&!4!4????#7 "5 " L
 mmM222 NN,	 " 
"3T"DV"DD	i66==,,778LbQ,,-ABII@$G ! 
 //?% I 	7' 3
 E  :x0s   AG6G	A*G6>G?8G67G8%G6GG6>G6G GA G 2G6G6G6G6G 
G3G..G33G6c                     U R                  US9nUb  [        U5      S:w  a  gUS   S   R                  SS 5      nUb  SU;  a  gUR                  S5      S   nU[        R
                  ;   $ )Nr  rr   Fr   r   r   /)get_model_listr  r1  r  r   )_known_custom_logger_compatible_callbacks)r  r   r   litellm_modelsplit_litellm_models        r   r  "Router._is_prompt_management_model  s|    ((E(:
ZA!5"1&67;;GTJ C}$<+11#6q9"g&W&WWWr   c           	        #    UR                  SS 5      nUc#  [        S0 S[        5       [        5       S.UED6u  pC[	        [
        U5      nU R                  USSS./UR                  SS 5      S9nU R                  XSS	9  US
   R                  5       nUR                  SS 5      nUR                  S5      =(       d    US
   R                  SS 5      nUR                  S5      =(       d    US
   R                  SS 5      n	UR                  SS 5      =(       d    US
   R                  SS 5      n
Ub  [        U[        5      (       d  [        SU S[        U5       35      eU	b/  [        U	[        5      (       d  [        SU	 S[        U	5       35      eUR                  UU[!        US9UU	U
S9u  nnn1 SknUR#                  5        VVs0 s H  u  pX;  d  M  X_M     nnn0 UEUEUEnXS'   X#S'   XCS'   XS'   XS'   XS'   U R%                  US9nUb  ['        U5      S:X  a/  UR                  S5        [(        R*                  " S0 UD6I S h  vN $ U R,                  " S0 UD6I S h  vN $ s  snnf  N$ N7f)Nrd  r  )r  	rules_objr  userpromptrB  r  r   r  r  r  r   r   	prompt_idprompt_variablesprompt_labelz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=)r  )r   r  non_default_paramsr  r  r  >   r  r  prompt_versionbitbucket_configdotprompt_configr  r  r  r   r  r{   )r1  rl   rk   rp   r   LiteLLMLoggingr  r  r  r  r  r   r  r   r  get_chat_completion_promptrn   r}  r
  r  r   r  r  )r  r   r  r  litellm_logging_objectprompt_management_deploymentr  r  r  r  r  optional_paramsprompt_management_paramskvfiltered_data_model_lists                    r   r  !Router._prompt_management_factory(  s     "(,A4!H!)-; .)6!&"2"4 	.*" "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 	++3 	, 	
 ,,<=BBD$/JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	
 zz.$7 $;W<

#nd
# 	 Jy#$>$><YKwtT]N_`  '
;KT0R0RDEUDVV]^bcs^t]uv  #==@O-% > 
		
$
  "ZZ\
)TQQ-NDAD\ 	 
 @M?V??w%z(>$%'{%5!"!-~))U);#k"2a"7JJ*+ ,,6v66677A&AAA#
 7As7   GI2I(I(A.I2	I.
I2#I0$I20I2r  c                     X#S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6nU$ ! [         a  nUeS nAff = f)Nr   r  r  r   r   rH  r{   )_image_generationr1  r   r   r   r  r  r  r  r   r  r  r  s         r   image_generationRouter.image_generation  s    
	#7O%8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL33=f=HO 	G	   A,A/ /
B 9A;;B c           	         Sn [         R                  " SU SU 35        U R                  USSS./UR                  SS 5      S9nU R	                  XSS	9  US
   R                  5       nU R                  UUS	9nU R                  U==   S-  ss'   U R                  US9  [        R                  " S0 0 UEUU R                  US.EUED6nU R                  U==   S-  ss'   [         R                  " SU S35        U$ ! [         aG  n	[         R                  " SU S[        U	5       S35        Ub  U R                   U==   S-  ss'   U	eS n	A	ff = f)Nr  #Inside _image_generation()- model: 
; kwargs: r  r  rB  r  r  r  r   rr   r  r  r  r  zlitellm.image_generation(model=r  r  r  r{   )r   r  r  r  r  r  ri  r   r  r   r)  r   r   r  r  r   r   )
r  r  r   r  r  r  r  r  r  r  s
             r   r'  Router._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C5C: :
EAEEc           
      `  #     X#S'   XS'   U R                   US'   UR                  SU R                  5      US'   U R                  X#S9  U R                  " S0 UD6I S h  vN nU$  N! [
         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)Nr   r  r  r   r  r  r{   )_aimage_generationr1  r   r  r  r  r  r  r@   r"  r#  r(  s         r   aimage_generationRouter.aimage_generation  s     	#7O%8*.*A*AF&'$*JJ}d>N>N$OF=!00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B.AA' A%A' $B.%A' '
B+15B&&B++B.c           	      .  #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XcS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)Nr-  r.  r  r  rB  r  r  r  r   r   rr   r/  re  rf  rc  z litellm.aimage_generation(model=r  r  r  r{   )r   r  r   rb  r  r  r  ri  r   r   r3  r   r  r  r  rj  rk  r   r  r  r   r   )r  r  r   r  r  r  r  r  r  r  rq  r  s               r   r2  Router._aimage_generation  s    
D	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&2:,>UV OqJ )
  . )=== *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG FB5G FG F#$F%	F#.F/F#3G >F!?G F=	G  F?!6G HG G F#F#!G #F:)F,*F:6G ?G 
HAHHHfilec           
      "  #     X#S'   XS'   U R                   US'   U R                  X#S9  U R                  " S0 UD6I Sh  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeSnAff = f7f)aB  
Example Usage:

```
from litellm import Router
client = Router(model_list = [
    {
        "model_name": "whisper",
        "litellm_params": {
            "model": "whisper-1",
        },
    },
])

audio_file = open("speech.mp3", "rb")
transcript = await client.atranscription(
model="whisper",
file=audio_file
)

```
r   r7  r  r  Nr  r{   )	_atranscriptionr  r  r  r  r  r@   r"  r#  )r  r7  r   r  r  r  s         r   atranscriptionRouter.atranscription	  s     .	#7O!6N*.*>*>F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		9   B;A A A BA 
B5BBBc           	      $  #    Un [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XcS9  US	   R                  5       nU R                  UUS9nU R                  U==   S
-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S
-  ss'   [         R$                  " SU S35        U	$  GN= N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S
-  ss'   UeS nAff = f7f)Nz!Inside _atranscription()- model: r.  r  r  rB  r  r  r  r   rr   )r7  r  r  re  rf  rc  zlitellm.atranscription(model=r  r  r  r{   )r   r  r   rb  r  r  r  ri  r   r   r:  r   r  r  r  rj  rk  r   r  r  r   r   )r  r7  r   r  r  r  r  r  r  r  rq  r  s               r   r9  Router._atranscriptionF	  s|    
B	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&/
|;RS OmF )
  . )=== *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAF< FB0F< FF< FF 	F)F*F.F< 9F:F< F8	F< F:6F< HF< F< FFF< F5$F'%F51F< :F< <
HAHHHinputvoicec           
        #     X$S'   X4S'   U R                  USSS./UR                  SS5      US9I Sh  vN nU R                  XS	9  US
   R                  5       nUS     U R                  R                  5        H+  u  pxXt;  a  XU'   M  US:X  d  M  XG   R                  U5        M-     U R                  XTSS9n	UR                  SS5      n
U
b  U	b  XR                  :w  a  SnOU	n[        R                  " S0 0 UESU0EUED6I Sh  vN nU$  N N! [         a:  n[        R                  " [        U U[         R"                  " 5       US95        UeSnAff = f7f)a  
Example Usage:

```
from litellm import Router
client = Router(model_list = [
    {
        "model_name": "tts",
        "litellm_params": {
            "model": "tts-1",
        },
    },
])

async with client.aspeech(
    model="tts",
    voice="alloy",
    input="the quick brown fox jumped over the lazy dogs",
    api_base=None,
    api_key=None,
    organization=None,
    project=None,
    max_retries=1,
    timeout=600,
    client=None,
    optional_params={},
) as response:
    response.stream_to_file(speech_file_path)

```
r?  r@  r  r  rB  r  Nr  r  r   r   r   r  rf  r  r  r  r{   )rb  r  r  r  r   r}  r   r  r1  r  r   aspeechr  r  r  r@   r"  r#  )r  r   r?  r@  r  r  r  r!  r"  r  r  r  r  r  s                 r   rB  Router.aspeech	  s    @4	#7O#7O#BB#)h?@$*JJ/Dd$K%	  C   J 00u0L./446DM3399;O !1I*_I$$Q' < &*%5%5%' &6 &" %jjD9O+*6#'E'EE#5$__ l  H OM>  		(,0#)(1(<(<(>'(	 G		sR   E1D DAD A4D DD ED D 
E5EEEc           
      ,  #     XS'   [         US'   U R                  US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[
        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7fNr   r?  r  r  r  r{   )
r?  _arerankr  r  r  r  r  r@   r"  r#  r  r   r  r  r  s        r   arerankRouter.arerank	  s     	#7O#F7O*.--F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BA A AA 
BA 
B5BBBc           	        #    S n [         R                  " SU SU 35        U R                  UUR                  SS 5      US9I S h  vN nU R	                  XBS9  US   R                  5       nUS   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEU R                  US	.EUED6I S h  vN nU R                  U==   S-  ss'   [         R                  " S
U S35        U$  N N9! [         aG  n[         R                  " S
U S[        U5       S35        Ub  U R                  U==   S-  ss'   UeS nAff = f7f)NzInside _rerank()- model: r.  r  )r   r  r  r  r   r   rr   )r  r  zlitellm.arerank(model=r  r  r  r{   )r   r  rb  r  r  r  ri  r   r   rH  r   r   r  r  r   r   )	r  r   r  r  r  r  r  r  r  s	            r   rF  Router._arerank	  s    
'	!''+E7*VHE  $BB$*JJ/Dd$K%  C   J
 //:/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O9  	!&&(4HQPWX %
+q0+G	sO   EA D C=A>D C?6D <E=D ?D 
EAEEEis_retryis_fallbackis_asyncc                    SUS./n XS'   X&S'   UR                  SU R                  5      US'   UR                  S0 5      R                  SU05        U R	                  UUUR                  SS 5      S	9nUS
   R                  5       n	U R                  R                  5        H+  u  pX;  a  XU
'   M  U
S:X  d  M  Xj   R                  U5        M-     [        R                  " S0 0 U	EX R                  S.EUED6$ ! [         a  nUeS nAff = f)Nr  rB  r   r  r   r   rH  r  r  r   )r  r  r{   )r1  r   r   r   r  r  r  r   r}  r   text_completionr   r  )r  r   r  rL  rM  rN  r  r  r  r  r!  r"  r  s                r   rP  Router.text_completion"
  s"    $78	#7O%8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399;O !1I*_I$$Q' < **s-r-rSgSg-rkq-rss 	G	s   B,C7 8>C7 7
DDDc           
        #    UR                  SS 5      b5  U R                  UUR                  S5      U R                  X4US9I S h  vN $  XS'   X&S'   U R                  US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  NG N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr  )r   r  r  rI  r  r   r  r  r  r  r{   )r1  r  r  atext_completion_atext_completionr  r  r  r  r  r@   r"  r#  )	r  r   r  rL  rM  rN  r  r  r  s	            r   rS  Router.atext_completionG
  s      ::j$'3//J/"&"7"7_ 0   	#7O%8*.*@*@F&'00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sH   ACBC;B BB CB 
C5CCCc           	      *  #     [         R                  " SU SU 35        [        U5      nU R                  USUS./UR	                  SS 5      US9I S h  vN nU R                  XSS9  US   R                  5       nUS	   nU R                  UUS9nU R                  U==   S
-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XTS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XTS9I S h  vN   U	I S h  vN n	U R"                  U==   S
-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S
-  ss'   UeS nAff = f7f)N#Inside _atext_completion()- model: r.  r  rB  r  r  r  r   r   rr   r/  re  rf  rc  zlitellm.atext_completion(model=r  r  r  r{   )r   r  r   rb  r  r  r  ri  r   r   rS  r   r  r  r  rj  rk  r   r  r  r   r   )r  r   r  r  r  r  r  r  r  r  rq  r  s               r   rT  Router._atext_completionl
  s~    C	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&1*=TU OoH )
  . )=== *  	!&&1%8LSQRVHT[\  &!+&G	   HAF? FB5F? 
FF? F!"F#	F!,F-F!1F? <F=F? F;	F? F=6F? HF? F? F!F!F? !F8'F*(F84F? =F? ?
H	AHHH
adapter_idc           
        #     X&S'   XS'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr   rZ  r  r   r   rH  r  r{   )_aadapter_completionr1  r   r   r   r  r  r  r  r@   r"  r#  )	r  rZ  r   rL  rM  rN  r  r  r  s	            r   aadapter_completionRouter.aadapter_completion
  s     	#7O#-< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA.A; 2A93A; 8C9A; ;
B?5B::B??Cc           	      *  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nU R                  XSS9  US	   R                  5       nUS
   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US.EUED6n	U R                  UUSS9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XTS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XTS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)Nz&Inside _aadapter_completion()- model: r.  r  zdefault textrB  r  r  r  r   r   rr   )rZ  r  r  re  rf  rc  z"litellm.aadapter_completion(model=r  r  r  r{   )r   r  r   rb  r  r  r  ri  r   r   r]  r   r  r  r  rj  rk  r   r  r  r   r   )r  rZ  r   r  r  r  r  r  r  r  rq  r  s               r   r\  Router._aadapter_completion
  s    C	!''8z&R  AH#BB#)nEF$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&4ZL@WX OoH )
  . )=== *  	!&&4UG;OPSTUPVxW^_  &!+&G	rY  c                 T   #    SSK Jn  UR                  " SU US.UD6I Sh  vN $  N7f)z
Helper function to make a search API call through the router with load balancing and fallbacks.
Reuses the router's retry/fallback infrastructure.
r   SearchAPIRouter)router_instancer  Nr{   )&litellm.router_utils.search_api_routerrc  async_search_with_fallbacks)r  r  r  rc  s       r   _asearch_with_fallbacksRouter._asearch_with_fallbacks  s=     
 	K$@@ 
 /
 
 
 	
 
s   (&(original_generic_functionc                 V   #    SSK Jn  UR                  " SU UUS.UD6I Sh  vN $  N7f)z
Helper function for search API calls - selects a search tool and calls the original function.
Called by async_function_with_fallbacks for each retry attempt.
r   rb  )rd  r   ri  Nr{   )re  rc  "async_search_with_fallbacks_helper)r  r   ri  r  rc  s        r   _asearch_with_fallbacks_helper%Router._asearch_with_fallbacks_helper"  s@      	K$GG 
 &?
 	
 
 	
 
s    )')c           
      \  #     XS'   X#S'   U R                   US'   U R                  XSS9  [        R                  " SU SU 35        U R                  " S
0 UD6I Sh  vN nU$  N! [
         a:  n[        R                  " [        U U[        R                  " 5       US	95        UeSnAff = f7f)
Helper function to make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
r   ri  r  litellm_metadatar   r  ru  z3Inside ageneric_api_call_with_fallbacks() - model: r.  Nr  r{   )(_ageneric_api_call_with_fallbacks_helperr  r   r  r  r  r  r  r@   r"  r#  )r  r   r  r  r  r  s         r   !_ageneric_api_call_with_fallbacks(Router._ageneric_api_call_with_fallbacks2  s     	#7O2C./*.*W*WF&'00CU 1  "''EeWJW]V^_ "??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B,AA% A#A% "B,#A% %
B)/5B$$B))B,r  c                     SU;   aU  US   (       aK  SSK Jn   U" UUR                  S5      UR                  S5      S9u  n    nUnUS   R	                  X'5      US'   U$ ! [         a    Un N'f = f)z
Add the deployment model to the endpoint for LLM passthrough route.

e.g for bedrock invoke users can pass endpoint as /model/special-bedrock-model/invoke
  it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke
endpointr   )rm   r/  r  )r   r/  r  )r   rm   r1  r  replace)r  r  r   r  rm   stripped_model_namer  replacement_model_names           r   ;_add_deployment_model_to_endpoint_for_llm_passthrough_routeBRouter._add_deployment_model_to_endpoint_for_llm_passthrough_routeQ  s     F:$6 1
4/?$(.

3H(I#ZZ
30,#Q1
 *=&
 "(
!3!;!;"F:   4)3&4s   -A A-,A-c           	      |  #    UR                  SS5      nSn [        U5      n U R                  UUUR                  SS5      UR                  SS5      S9I Sh  vN nU R                  XsUS	9  US
   R                  5       n	U	S   n
U R                  U
==   S-  ss'   U R                  X1U
S9  U" S0 0 U	ESU R                  0EUED6nU R                  UUSS9nUb]  [        U[        R                  5      (       a>  U ISh  vN    U R                  XvS9I Sh  vN   UI Sh  vN nSSS5      ISh  vN   O!U R                  XvS9I Sh  vN   UI Sh  vN nU R                  U
==   S-  ss'   [         R"                  " SU
 S35        U$  GN3! [         a&  nU(       a  U" SSU0UD6I Sh  vN  s SnA$ UeSnAff = f N N N N! , ISh  vN  (       d  f       N= f N N! [         aG  n[         R"                  " SU S[%        U5       S35        Ub  U R&                  U==   S-  ss'   UeSnAff = f7f)ro  passthrough_on_no_deploymentFrs  r  Nr  )r   r  r  r  r   r  r   rr   )r  r   r  r  re  rf  rc  z'ageneric_api_call_with_fallbacks(model=r  r  r  r{   )r  r   rb  r1  r  r  r  r   rz  r   r  r  r  rj  rk  r   r   r  r   r   )r  r   ri  r  r}  r  r  r  r  r  r  r  rq  s                r   rr  /Router._ageneric_api_call_with_fallbacks_helperp  s     (.zz2PRW'X$;F	@H
#'#F#F#)#ZZ
D9(.

3H$(O	 $G $ 
 //%M 0  ./446DgJZ(A-(LLz M  1 t33 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&9*E\] Oy  /!:!Q!Q&!QQQQD )
  . )=== *  	!&&9%@TUXYZU[T\\cd  &!+&G	s  H<G( 5F FF B"G(  GG( G
G	G
"G#G
'G( 2G3G( 
G$	G( G&6G( H<F 
F?F:-F0.F:2F?3G( 7H<8F::F??G( G
G
G( 
G!GG!G( &G( (
H92AH44H99H<c           
      @   UR                   n [        R                  " SU SU SU 35        U R                  UUR	                  SS5      UR                  SS5      S9nU R                  XSSS	9  US
   R                  5       nUS   nU R                  U==   S-  ss'   SU;   a!  US   (       a  US   R                  X5      US'   XsS'   U R                  US9   [        US   S9u  p  nU" S0 0 UEU	U R                  S.EUED6n
U R                  U==   S-  ss'   [        R                  " U SU S35        U
$ ! [         a    Sn	 N^f = f! [         aI  n[        R                  " U SU S[!        U5       S35        Ub  U R"                  U==   S-  ss'   UeSnAff = f)aV  
Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
Args:
    model: The model to use
    original_function: The handler function to call (e.g., litellm.completion)
    **kwargs: Additional arguments to pass to the handler function
Returns:
    The response from the handler function
z&Inside _generic_api_call() - handler: z	, model: r.  r  Nr  r  generic_api_callr  r   r   rr   rv  r  rx  )r/  r  z(model=r  r  r  r{   )r|   r   r  r  r1  r  r  r  r   rw  r  rm   r  r   r   r  r   r   )r  r   r  r  handler_namer  r  r  r  r/  r  r  s               r    _generic_api_call_with_fallbacks'Router._generic_api_call_with_fallbacks  s    )114	!''8iPUwV`ag`hi 66J5$*JJ/Dd$K 7 J
 //%DV 0  ./446DgJZ(A-( V#z(:%+J%7%?%?%Rz"(7O 11Z1H+/?d7m/T,1 ) +>#33 	H z*a/*!&&.
|3JK O!  +&*#+"  	!&&.w.B3q6('R  &!+&G	s>   C	E
 D8 (AE
 8EE
 EE
 

FAFFc                     XS'   X$S'   U R                   US'   UR                  SU R                  5      US'   UR                  S0 5      R	                  SU05        U R
                  " S0 UD6nU$ ! [         a  nUeS nAff = f)Nr   r?  r  r   r   rH  r{   )
_embeddingr1  r   r   r   r  r  r  r   r?  rN  r  r  r  s          r   	embeddingRouter.embedding  s    		#7O#7O*.//F&'$*JJ}d>N>N$OF=!j"-44mU5KL33=f=HO 	G	r+  c           	         S n [         R                  " SU SU 35        U R                  UUUR                  SS 5      S9nU R	                  XSS9  US   R                  5       nUS   nU R                  XSSS	9nUR                  S
S 5      nUb  Ub  XR                  :w  a  S n	OUn	U R                  U==   S-  ss'   U R                  US9  [        R                  " S0 0 UEUU R                  U	S.EUED6n
U R                  U==   S-  ss'   [         R                  " SU S35        U
$ ! [          aG  n[         R                  " SU S[#        U5       S35        Ub  U R$                  U==   S-  ss'   UeS nAff = f)NzInside embedding()- model: r.  r  )r   r?  r  r  r   r   syncrf  r  rr   r  r?  r  r  zlitellm.embedding(model=r  r  r  r{   )r   r  r  r  r  r  r  r1  r  r   r  r   r  r   r   r  r  r   r   )r  r?  r   r  r  r  r  r  r  r  r  r  s               r   r  Router._embedding  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:/U./446DgJ%)%5%5%& &6 &" %jjD9O+*6#'E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D"D' '
E81AE33E8c           
      "  #     XS'   X$S'   U R                   US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R
                  " [        U U[        R                  " 5       US95        UeS nAff = f7frE  )	_aembeddingr  r  r  r  r  r@   r"  r#  r  s          r   
aembeddingRouter.aembeddingN  s     	#7O#7O*.*:*:F&'00u0L!??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r<  c           	      &  #    S n [         R                  " SU SU 35        [        U5      nU R                  UUUR	                  SS 5      US9I S h  vN nU R                  XcS9  US   R                  5       nUS   nU R                  UUS9nU R                  U==   S-  ss'   [        R                  " S0 0 UEUU R                  US	.EUED6n	U R                  UUS
S9n
U
b]  [        U
[        R                  5      (       a>  U
 IS h  vN    U R!                  XeS9I S h  vN   U	I S h  vN n	S S S 5      IS h  vN   O!U R!                  XeS9I S h  vN   U	I S h  vN n	U R"                  U==   S-  ss'   [         R$                  " SU S35        U	$  GNB N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [&         aG  n[         R$                  " SU S[)        U5       S35        Ub  U R*                  U==   S-  ss'   UeS nAff = f7f)NzInside _aembedding()- model: r.  r  )r   r?  r  r  r  r   r   rr   r  re  rf  rc  zlitellm.aembedding(model=r  r  r  r{   )r   r  r   rb  r  r  r  ri  r   r   r  r   r  r  r  rj  rk  r   r  r  r   r   )r  r?  r   r  r  r  r  r  r  r  rq  r  s               r   r  Router._aembeddingg  s{    
B	!''/wjI  AH#BB$*JJ/Dd$K%	  C   J //:/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&+J<7NO OmF )
  . )=== *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAF= FB5F= F	F= F F!	F*F+F/F= :F;F= F9	F= F;6F= HF= F= FFF= F6%F(&F62F= ;F= =
HAH		HHc           
      X  #     XS'   U R                   US'   UR                  SU R                  5      US'   U R                  XS9  U R                  " S0 UD6I S h  vN nU$  N! [
         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)Nr   r  r   r  r  r{   )_acreate_filer1  r   r  r  r  r  r  r@   r"  r#  rG  s        r   acreate_fileRouter.acreate_file  s     
	#7O*.*<*<F&'$*JJ}d>N>N$OF=!00u0L!??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B*AA# A!A#  B*!A# #
B'-5B""B''B*c                   ^ ^^#     SSK Jn  [        R                  " SU ST 35        [	        T5      mT R                  USSS./TR                  SS 5      TTS	9I S h  vN nS
[        S[        4UUU 4S jjn/ n[        U[        5      (       a  UR                  U" U5      5        O U H  nUR                  U" U5      5        M     [        R                  " U6 I S h  vN n[        U5      S:X  a  [        S5      eU" XHS9n	[        [        US   5      n
U	U
R                   S'   U
$  N NJ! [         aJ  n[        R"                  " SU ST S[%        U5       S35        Ub  T R&                  U==   S-  ss'   UeS nAff = f7f)Nr   )add_model_file_id_mappingsrW  r.  r  files-api-fake-textrB  r  r   r  r  r  r  r  r   c           	        >#    SSK Jn  U" T5      nTR                  U USS9  U S   R                  5       nUS   nTR	                  U US9nTR
                  U==   S-  ss'   [        US   S	9u  pg  n[        [        [           TR                  S
5      5      n	[        [        [           TR                  S5      5      n
U
(       a  U	(       d  [        S5      e[        U	S9nU(       a  [        U
US9n
XS'   [        R                   " S0 0 UEUTR"                  US.EUED6nTR%                  U USS9nUb^  ['        U[(        R*                  5      (       a?  U IS h  vN    TR-                  U TS9I S h  vN   UI S h  vN nS S S 5      IS h  vN   O"TR-                  U TS9I S h  vN   UI S h  vN nTR.                  U==   S-  ss'   [0        R2                  " SU S35        U$  N Nv Nn N`! , IS h  vN  (       d  f       NR= f Nc N[7f)Nr   safe_deep_copyr  r  r   r   r  rr   rx  purposer7  z2file and file_purpose are required for create_file)r  )file_contentnew_model_namer/  r  r  re  rf  rc  litellm.acreate_file(model=r  r{   )'litellm.litellm_core_utils.core_helpersr  r  r  ri  r   rm   r   r   rJ   r1  rH   r  r1   r0   r   r  r   r  r  r  rj  rk  r   r   r  )r  r  kwargs_copyr  r  r  stripped_modelr/  r  r  r7  replace_model_in_jsonl_boolr  rq  r  r  r  s                 r   create_file_for_deployment8Router._acreate_file.<locals>.create_file_for_deployment  sh    R,V433)&"0 4 
 ""2388:!']
#BB)&  C     ,1, =Mw-=9Q
 x(:;VZZ	=RSHY/F1CD7#L  /L#/+ /1%)'5D
 +/'"// /B#'#7#7".	
 & !% 0 0)& 7 !1 ! !,!7#4#42 2  -} #II'1DT J    *2>  -}} EE#-@P F    &.~H"":.!3.%**1*=TU  )  -
 $2  -}}}  .s   EHG&HG.1G(2	G.;G*<G. HG,H$H%	H.H
/8H(G.*G.,H.H4G75HH
HNo healthy deployments found.)r   r[  model_file_id_mappingr  , r  r  rr   )!litellm.router_utils.common_utilsr  r   r  r   async_get_healthy_deploymentsr  r  rI   r  r   r  r  r  r  r   r  	exceptionr   r   )r  r   r  r  r   r  tasksr  r[  r  returned_responser  r  s   ` `         @r   r  Router._acreate_file  s    
}	T!''5eWJvhO  AH(,(J(J#)6KLM$*JJ/Dd$K%!1 )K ) #Q T Q FV Q  Q f E-t4478KLM"5JLL!;J!GH #6 &nne44I9~" ?@@$>$7%! !%%5y| D & ,,' %$_#F 5  	!++-eWBvh>RSVWXSYRZZab  &!+&G	sP   FAD+ D'A?D+ D)AD+ &F'D+ )D+ +
E?5AE::E??Fc           
      n  #     XS'   U R                   US'   UR                  SU R                  5      US'   [        SS9nU R	                  UUUS9  U R
                  " S0 UD6I S h  vN nU$  N! [         a:  n[        R                  " [        U U[        R                  " 5       US95        UeS nAff = f7f)	Nr   r  r   _acreate_batchr  rq  r  r{   )r  r1  r   r/   r  r  r  r  r  r@   r"  r#  )r  r   r  ru  r  r  s         r   acreate_batchRouter.acreate_batchJ  s     
	#7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 "??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B5A!A. %A,&A. +B5,A. .
B285B--B22B5c                 R  #     [         R                  " SU SU 35        [        U5      nU R                  USSS./UR	                  SS 5      US9I S h  vN nUS   R                  5       nUS	   nU R                  XBS
S9  U R                  UUS9nU R                  U==   S-  ss'   [        US	   S9u  p  n[        R                  " S0 0 UEU	U R                  US.EUED6n
U R                  UUSS9nUb]  [        U[        R                   5      (       a>  U IS h  vN    U R#                  XCS9I S h  vN   U
I S h  vN n
S S S 5      IS h  vN   O!U R#                  XCS9I S h  vN   U
I S h  vN n
U R$                  U==   S-  ss'   [         R&                  " SU S35        U
$  GNS N Nx Np Nb! , IS h  vN  (       d  f       NU= f Nf N^! [(         aJ  n[         R*                  " SU SU S[-        U5       S35        Ub  U R.                  U==   S-  ss'   UeS nAff = f7f)Nz Inside _acreate_batch()- model: r.  r  r  rB  r  r  r   r   r  r  r  rr   rx  r  re  rf  rc  zlitellm.acreate_batch(model=r  zlitellm._acreate_batch(model=r  r  r  r{   )r   r  r   rb  r  r  r  ri  r   rm   r   r  r   r  r  r  rj  rk  r   r  r  r  r   r   )r  r   r  r  r  r  r  r  r  r/  r  rq  r  s                r   r  Router._acreate_batchi  s    
I	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J ./446DgJ//%DT 0   >>% ? L Z(A-( ,<$w-+P(AAq,, +>#33*	
 H !,,%3 - M (Zw00. . )= EE#- F    &.~H )== AA) B    "*>z*a/*!&&.zl:QR O{R )
  . )=== *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   H'AG F'CG F*G F23F,4	F2=F.>F2G F0G %G&	G /G06G &H''G *G ,F2.F20G 2G	8F;9G	G G 
H$AHH$$H'c                   ^ ^^	#     [        T5      nUb0  T R                  USSS./TR                  SS5      TUS9I Sh  vN nOT R                  5       nUc  [	        S5      e/ m	S[
        4UU	U 4S	 jjnUbh  [        U[        5      (       aS  [        U5      S
:  aD  [        R                  " U Vs/ s H  nU" [        [
        U5      5      PM     snSS06I Sh  vN nOBUb4  [        U[        5      (       a  U" [        [
        U5      5      I Sh  vN nO[	        S5      eUbN  [        U[        5      (       a  U$ [        U[        5      (       a"  U H  n[        U[        5      (       d  M  Us  $    T	(       a  T	S
   e[	        SR                  T	5      5      e GNZs  snf  N N! [         a:  n[        R                  " [!        T T["        R$                  " 5       US95        UeSnAff = f7f)zh
Iterate through all models in a model group to check for batch

Future Improvement - cache the result.
Nr  zretrieve-api-fake-textrB  r  r  Router not yet initialized.r  c                 4  >#     SSK Jn  U S   R                  S5      nU S   R                  5       nUR                  S5      nUc  [	        SU  35      eU(       d  [        US9u  pT  nU" T	5      nTR                  [        [        U 5      USS	9  UR                  SS 5        UR                  SS 5        [        R                  " S
0 0 UESU0EUED6I S h  vN $  N! [         a0  nSS KnUR                  " 5         T
R                  U5         S nAg S nAff = f7f)Nr   r  r   r   r/  z2Model not found in litellm_params for deployment: rx  aretrieve_batchr  r{   )r  r  r1  r  r  rm   r  r   r  r  r   r  r"  	print_excr   )r  r  r   r  r/  r  
new_kwargsr  r"  r  receieved_exceptionsr  s            r   try_retrieve_batch2Router.aretrieve_batch.<locals>.try_retrieve_batch  sC    $ V&'78<<WEE%&67<<>D*.((3H*I'}'PQ[P\] 
 /7G"'841 "0!7J77#'j#9)&7 8 
 NN#8$?HH2D9!(!8!8 ""13F )"    !  $'')(//2 sA   DCC CC DC 
D%&DDDDr   return_exceptionsTr  z7Unable to find batch in any model. Received errors - {}r  )r   r  r  r
  r  rS   r  r  r  r  r  r   r  rc   r  r  r@   r"  r#  )
r  r   r  r  filtered_model_listr  resultsr  r  r  s
   ` `      @r   r  Router.aretrieve_batch  s    l	@H  <<'-:RST(.

3H$(O#)%5 =   $ '+&9&9&;#"* =>>#% % 5H %  % R $/2D99+,q0 ' &9%8E +40CU+KL%8!
 '+!  %0Z#T6 6 !3,.AB!    ?@@ "g|44"N..")%fl;;#)M #*
 $*1-- IPP( sx2  		(,0#)(1(<(<(>'(	 G		s|   G,8F% F A0F% 0 F
F% F!6F% F#)F% ;G,<-F% -F% 0G,1)F% F% #F% %
G)/5G$$G))G,c                   ^#    U R                  US9nUc  [        S5      eS[        4U4S jjn[        R                  " U Vs/ s H
  o" U5      PM     sn6 I Sh  vN nS/ SSSS.nU Hr  nUc  M  US	   c   [        US	5      (       a  [        US	5      US	'   [        US
5      US
'   US   R                  UR                  5        [        USS5      SL d  Mm  SUS'   Mt     U$ s  snf  N7f)zA
Return all the batches across all deployments of a model group.
r  Nr  r   c                 |   >#     [         R                  " S0 0 U S   ETED6I S h  vN $  N! [         a     g f = f7f)Nr   r{   )r   alist_batchesr  r  s    r   r  0Router.alist_batches.<locals>.try_retrieve_batch>  sS     $22 ;/0;F;     s(   < , *, <, 
9<9<r  F)objectr  first_idlast_idhas_morer  r  r  r  T)	r
  r  rS   r  r  rP  rM  extendr  )r  r   r  r  r  r  final_resultsr  s     `     r   r  Router.alist_batches1  s     #11U1C&9::	,? 	  5HI5HE '5HI
 

 
 F! ,49T9T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-  / J
s+   A C)C"C)C'C)2A!C)C)r/  c                   #    U R                  UR                  SS5      US9  UR                  S5      (       aq  U R                  US   S9(       aZ  U R                  US   US9I S h  vN nUS   S   US'   US   R	                  5       nU R                  UUS9  UR                  U5        U" S0 UD6I S h  vN $  NS N7f)	Nr   r  r  r  )r   r  r   r  r{   )r  r1  r
  rb  r  r  r   )r  r  r/  r  r  r  s         r   )_pass_through_moderation_endpoint_factory0Router._pass_through_moderation_endpoint_factoryd  s      	,,**Wb) 	- 	
 ::g4#6#6&/#6#R#BBWo%  C   J ))9:7CF7O./446D//% 0  MM$&0000 1s%   A$B>&B:'AB>5B<6B><B>rT  )>
assistantsrR  rU  rX  r]  r[  r\  r^  rY  rZ  r`  ra  rb  rc  rd  re  rf  rg  r_  rV  r  r  r  rr  rs  r~  r  r  r  r  r  rt  ru  r  r  r  r  r  r  r  r  ro  rq  rW  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  c                   ^ ^^ TS;   a)    SS[         [           S[         [           4UU 4S jjjnU$ TS;   a(    SS[         [           S[         [           4U4S jjjnU$   SS[         [           S[         [           4UUU 4S jjjnU$ )	z
Creates appropriate wrapper functions for different API call types.

Returns:
    - A synchronous function for synchronous call types
    - An asynchronous function for asynchronous call types
)r[  r  r  rt  ru  r  rq  r  r  r  r  r  r  r  r  r  r/  r  c                 ,   > TR                   " SST0UD6$ )Nr  r{   )r  )r/  r  r  r  r  s      r   sync_wrapper-Router.factory_function.<locals>.sync_wrapper  s(    
 << &7;A r   )r  r  r  r  r  r  c                    > T" SU US.UD6$ )Nr/  r  r{   r{   )r/  r  r  r  s      r   vector_store_file_sync_wrapper?Router.factory_function.<locals>.vector_store_file_sync_wrapper  s'    
 ) (;!  r   c                   >#    TS:X  a  TR                   " STU US.UD6I S h  vN $ TS:X  a  TR                  " SST0UD6I S h  vN $ TS;   a  TR                  " SST0UD6I S h  vN $ TS;   a  TR                  " STU S.UD6I S h  vN $ TS;   a  TR                  " SST0UD6I S h  vN $ TS	:X  a  TR                  " STS
S.UD6I S h  vN $ TS;   a  TR
                  " SST0UD6I S h  vN $ TS;   a  TR                  " STU S.UD6I S h  vN $ TS;   a  TR                  " STU US.UD6I S h  vN $ g  GN N N N N Ny NY N8 N7f)Nr  )r  r/  r  rR  r  r  )r~  r  r  r  r  r  )r  r/  )rU  rX  r`  ra  rb  rc  rd  re  rf  rV  r  r  r  r  r  r  r  r  r  r  r  r  rg  T)r  r}  )r\  r]  r^  r_  )rr  rs  )rY  rZ  r{   ))_pass_through_assistants_endpoint_factoryr  rg   _init_vector_store_api_endpointsrs  _init_responses_api_endpoints)r/  r  r  rT  r  r  s      r   async_wrapper.Router.factory_function.<locals>.async_wrapper  s:    
 L(!KK &7(;! 	   l*!KK &7;A   33!99 &7      "BB &7(;   
   0 "CC &7   66!CC &715   
    "?? &7      "BB &7(;   
 ??!CC &7(;! 	   @c:
s    ED>!EE!E'E("E
E!E,E-"EE	!E1E2"EE#E8E9EEEEE	EEEENN)r   r   r   )r  r  rT  r  r  r  s   ```   r   rF  Router.factory_function~  s    X  
 
& 6:(,%-c]     
 
 6:(,	%-c]	 	 	 21 26$(]	!)#]	SM]	 ]	~ r   c                 P   #    U(       a
  SU;  a  X#S'   U" S0 UD6I Sh  vN $  N7f)z:
Initialize the Vector Store API endpoints on the router.
r/  Nr{   r{   )r  r  r/  r  s       r   r  'Router._init_vector_store_api_endpoints_  s/      #8#F,?()&00000s   &$&c                    #    SSK Jn  UR                  UR                  S5      5      nUb  XBS'   U R                  " SSU0UD6I Sh  vN $  N7f)z
Initialize the Responses API endpoints on the router.

GET, DELETE, CANCEL Responses API Requests encode the model_id in the response_id, this function decodes the response_id and sets the model to the model_id.
r   )ResponsesAPIRequestUtilsresponse_idNr   r  r{   )litellm.responses.utilsr  get_model_id_from_response_idr1  rs  )r  r  r  r  model_ids        r   r  $Router._init_responses_api_endpointsl  sc      	E+IIJJ}%
 &7O;; 
/

 
 	
 
s   AAAAr  c                    #    UcF  U R                   b.  U R                   S   nUR                  U R                   S   5        O[        S5      eU" SX#S.UD6I Sh  vN $  N7f)z@Internal helper function to pass through the assistants endpointNr/  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`r  r{   )r   r   r  )r  r  r/  r  r  s        r   r  0Router._pass_through_assistants_endpoint_factory  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3
FL
 
 	
 
s   AA AA r  rG  rH  c	                   #    [         R                  " S[        R                  " 5        35        Un	Sn
UR	                  S5      nSnUSL d  Uc  UeU U	S.UEnSU;  a  U R
                  US'   SU;  a  S	US'    [         R                  " S
5        [        US9nU(       a)  UR                  UUS.5        [        U0 UD6I Sh  vN nU$ [        U[        R                  5      (       a  Ub>  U R                  UUS9nUc  U	eUR                  UUS.5        [        U0 UD6I Sh  vN nU$ SR                  XdU5      n[         R                  " SR                  U5      S9  U=R                  SR                  U5      -  sl        O[        U[        R                   5      (       a  Ub>  U R                  UUS9nUc  U	eUR                  UUS.5        [        U0 UD6I Sh  vN nU$ SR                  XeU5      n[         R                  " SR                  U5      S9  U=R                  SR                  U5      -  sl        Ub  Ub  [         R                  " SU 35        [#        U[%        [&        U5      S9u  n
nU
c  Ub  UU   S   n
U
cJ  [         R                  " SU SU 35        [)        U	S5      (       a  U	=R                  SU SU 3-  sl        U	eUR                  U
US.5        [        U0 UD6I Sh  vN nU$ [)        U	S5      (       aX  U	=R                  SR                  UU
5      -  sl        [3        U5      S	:  a$  U	=R                  SR                  U5      -  sl        U	e GN GNF GN Nz! [*         at  n[-        U5      n[         R.                  " SR                  ['        U5      [        R                  " 5       [1        U US9I Sh  vN  5      5        ['        U5      n SnANSnAff = f7f)z4
Common utilities for async_function_with_fallbacks
	TracebackNr   r  T)litellm_routerr  r   fallback_depthr   zTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   rH  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                         to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                         to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=r  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}r  r  z<. Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})r   r  r"  r#  r1  r   r  r;   r   r=   r  r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr  r  r  r<   r   r   rP  r  r   rT  r8   r  )r  r  rG  r   r   r   rH  rI  r  r  r  r  fallback_failure_exception_strro  is_non_standard_fallback_formatr  #context_window_fallback_model_grouperror_message#content_policy_fallback_model_groupgeneric_fallback_idxnew_exceptionr  s                         r   rO  1Router.async_function_with_fallbacks_common_utils  s     	##i	0D0D0F/G$HI#.4jj.A)+&$(<(DG #"4
 
 ,.,0,>,>L)</-.L)*L	@!&&'FG /R#/+ /##090D "4""" 
  !W??@@+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %m  %t  %t#y%M *..77=v)8 II}!==IAwBBCC+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %l  %s  %s#y%M *..77=v)8 II}!==I$)@%++.Fyk,RS -' $S+ 6(( (/4H4T+45I+J3+O('/)..QR]Q^^jktjuv 19==*228op{o|  }I  JS  IT  7U  U2,,##0D0D "4""" 
   %y11&&*i*p*p$+ & 12Q6"**4;;6* ! W2 H Z  	@@H!'' k  r  r&((*I04)9  	 .1-?*	@s   A5O/8AM.  M#M. O/AM.  M&!M. &O/'B3M. M)M.  O/!DM. 1M,2M. 7A,O/#M. &M. )M. ,M. .
O,8AO'	O
O'"O/'O,,O/c                 ~  #    UR                  S5      nUR                  SS5      nUR                  SU R                  5      nUR                  SU R                  5      nUR                  SU R                  5      nUR                  SS5      n U R                  UUUUUS	9  Ub  U R                  " U0 UDSU0D6I Sh  vN n	OU R                  " U0 UD6I Sh  vN n	[        R                  " S
U	 35        [        U	SS9n	U	$  ND N+! [         a+  n
U R                  U
UUUUUUU5      I Sh  vN  s Sn
A
$ Sn
A
ff = f7f)zg
Try calling the function_with_retries
If it fails after num_retries, fall back to another model group
r   rG  Fr   r   r   mock_timeoutN)r  rH  r   r   r   zAsync Response: r   )r  attempted_fallbacks)r1  r  r   r   r   _handle_mock_testing_fallbacksasync_function_with_retriesr   r  r-   r  rO  )r  rI  r  rH  rG  r   r   r   r  r  r  s              r   r  $Router.async_function_with_fallbacksW  sr     &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7	//'#)A)A 0  '!%!A!A"#"2>"  "&!A!A4!R6!RR!''*:8*(EF7!$%H O S  
	HH!((	 	 	 	
	sf   B
D=/D <D=D D(D  D=D D 
D:D5*D-+D5/D:0D=5D::D=c                    [         R                  " U5      nUR                  b+  UR                  SL a  [        R                  " USSU SU 3S9eUR
                  b+  UR
                  SL a  [        R                  " USSU SU 3S9eUR                  b,  UR                  SL a  [        R                  " USSU SU 3S9egg)	a  
Helper function to raise a litellm Error for mock testing purposes.

Raises:
    litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
    litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
    litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
NTr  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=)r   r  r  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)	rU   from_kwargsmock_testing_fallbacksr   InternalServerErrormock_testing_context_fallbacksr  %mock_testing_content_policy_fallbacksr  )r  r  rH  r   r   r   mock_testing_paramss          r   r  %Router._handle_mock_testing_fallbacks  s     6AA&I66B#::dB--!=k]Jmnwmxy   >>J#BBdJ44!=k] K..F-GI   EEQ#IITQ55!=k] K..F-GI  R Rr   c                   #    [         R                  " S5        UR                  S5      nUR                  SU R                  5      n[	        U5      nUR                  SU R
                  5      nUR                  SU R                  5      nUR                  S5      nUR                  S5      n	UR                  SUR                  S	5      5      =(       d    0 n
S
U
;   aI  [        U
S
   [        5      (       a1  U R                  U
S
   S9nUb  U
R                  S[        U5      05        [         R                  " SU SU	 35         U R                  XS9  U R                  " U/UQ70 UD6I S h  vN n[        USS S9nU$  N! [          GaN  nS nUn[#        USS 5      nUb  [        U[$        5      (       a  Un	 U R'                  UR                  S5      =(       d    SUS9I S h  vN  u  nnU R)                  UUUUUUS9  U R*                  c  U R,                  b#  U R/                  XR                  S5      S9nUb  Un	U	S:  a  U R1                  X/S9nOe [         R                  " SU	 35        U R3                  UU	U	UUS9n[4        R6                  " U5      I S h  vN    [9        U	5       H  n U R                  " U/UQ70 UD6I S h  vN  n[:        R<                  " U5      (       a  UI S h  vN  n[        UUS-   U	S9nUs  s S nA$ ! [          a  nU R1                  X-S9nX-
  nUR                  S5      nUb  U R'                  UUS9I S h  vN  u  nnO/ nU R3                  UUU	UUS9n[4        R6                  " U5      I S h  vN     S nAM  S nAff = f   [?        U5      [@        RB                  ;   a  [E        USU	5        [E        USU5        UeS nAff = f7f)Nz#Inside async function with retries.r  r   r   r   r   r   rp  r   rH  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - )rH  r  r   )r  attempted_retriesr   r  r  )rT  r   all_deploymentsr   regular_fallbacksr   )r  rH  )r  r  z#Retrying request with num_retries: )r  remaining_retriesr   r   r  rr   r   )#r   r  r  r   r   r   r   r1  r  r   r
  r   r  %_handle_mock_testing_rate_limit_error	make_callr.   r  rM  r   r  should_retry_this_errorr   r   r>   	log_retry_time_to_sleep_before_retryr  r  r  r    is_async_callabler   r   LITELLM_EXCEPTION_TYPESrS  )r  rI  r  r  r   r  r   r   rH  r   	_metadatar   r  r  current_attemptr  deployment_num_retriesr  _all_deployments_retry_policy_retriesr   r  _modelr  _timeouts                            r   r  "Router.async_function_with_retries  sK    ##$IJ"JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**%7J9OPVTV	I%*Y}5Ms*S*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
r	%66' 7  "^^,=OOOOH4!QDH O	 P
  h	%"O!"%,Qt%D"%1j&7 7 5 99jj)/R!1 :   $  (($8 0)A"+)A )  !!-00< )-(N(N0jj>Q )O )% )4"7KQvL!''5k]C ::$"-'$8 0 ; K --,,,#(#5%2%)^^4E%W%WPV%WWWH(::   *2>><!)*9A*=$/ H
 $O  2!^^6^?F(3(E%,2JJw,?F) #'"E"E"(-= #F #  0 02,#??,*;$/,@(8  @  H "--111-2! $6P &'7+J+JJ*M;G*M?K$$Qh	%s   D>O(E= )E;*E= :O;E= =OAOG" B5OJO*LK%L)K,*L>O OO
N:N
M
7NN
NON>OOOc                    #    UR                  S5      nU" U0 UD6n[        R                  " U5      (       d  [        R                  " U5      (       a
  UI Sh  vN nU R                  XTS9I Sh  vN nU$  N N7f)zN
Handler for making a call to the .completion()/.embeddings()/etc. functions.
r   N)r  rH  )r1  r    r  inspectisawaitableset_response_headers)r  r  rI  r  rH  r  s         r   r  Router.make_callF  s      jj)$d5f5..x88G<O<O=
 =
 &~H22 3 
 
  &
s$   AA9A5A9.A7/A97A9c                 @   UR                  SS5      nU R                  US9nSnUb;  [        U5      S:X  a,  [        [        [
           US   S   R                  S5      5      nUb:  USL a4  [        R                  " S	U 35        [        R                  " US
SU S3US9egg)z
Helper function to raise a mock litellm.RateLimitError error for testing purposes.

Raises:
    litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
mock_testing_rate_limit_errorNr  rr   r   r   r   TzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r  r  z , to trigger a rate limit error.)r   r  r  r   )r  r
  r  r   r   r   r1  r   r  r   RateLimitError)r  r  rH  r+  available_modelsr   s         r   r  ,Router._handle_mock_testing_rate_limit_errorW  s     9?

+T9
%  ..+.F%)'C0@,AQ,F/23CDHHWK
 *5-5!&&fgrfst ((!=k]Jjk'	  6 6r   rT  r   r  r  c                 <   SnUb   [        U[        5      (       a  [        U5      nSnUb   [        U[        5      (       a  [        U5      n[        U[        R                  5      (       a  Ub  Ue[        U[        R
                  5      (       a  Ub  Ue[        U[        R                  5      (       a  Ue[        U[        R                  5      (       a  US::  a  Ub  [        U5      S:  a  Ue[        U[        R                  5      (       a	   US::  a  UeUS::  a  Ueg)aE  
1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

2. raise an exception for RateLimitError if
    - there are no fallbacks
    - there are no healthy deployments in the same model group
r   rr   T)
r  r  r  r   r  r  NotFoundErrorr   r,  AuthenticationError)	r  rT  r   r  r   r   r  _num_healthy_deployments_num_all_deploymentss	            r   r  Router.should_retry_this_errorz  s   " $% *z:Mt/T/T'*+>'?$ &:ot+L+L#&#7  ug@@AA(4K ugAABB(4KeW2233KeV2233(A-%1)*Q.eV7788
 %) $q(Kr   c                 4    [        U R                  /UQ70 UD6$ )zf
Sync wrapper for async_function_with_fallbacks

Wrapped to reduce code duplication and prevent bugs.
)r   r  )r  rI  r  s      r   r  Router.function_with_fallbacks  s     "$"D"DVtVvVVr   c                 r    Uc  gSnU H*  n[        UR                  5       5      S   U:X  d  M%  XB   n  U$    U$ )a  
Returns the list of fallback models to use for a given model group

If no fallback model group is found, returns None

Example:
    fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
    model_group = "gpt-3.5-turbo"
    returns: ["gpt-4"]
Nr   )r  keys)r  r   rH  r  rU  s        r   r  /Router._get_fallback_model_group_from_fallbacks  sO     48DDIIK #{2'+'8$##	  $#r   c                     U R                   c  gU R                    HT  n[        U[        5      (       d  M  SU;   d  M"  US   n[        U[        5      (       d  M>  [	        U5      S:  d  MO  US   s  $    g)zH
Returns the first model from the default_fallbacks list, if it exists.
Nr   r   )r   r  r  r  r  )r  fallbackdefault_lists      r   _get_first_default_fallback"Router._get_first_default_fallback  sc     >>!H(D))cXo'}lD11c,6G!6K'?*	 '
 r   r  c                    Ub  [        U5      S:X  a  O(Ub%  [        U[        5      (       a  [        U5      S:  a  gSn[        US5      (       a1  [        UR                  S5      (       a  UR                  R
                  n[        US5      (       a  UR                  nUb#  [        R                  " UUUU R                  S9nU$ [        R                  " UUU R                  S9nU$ )	z
Calculate back-off, then retry

It should instantly retry only when:
    1. there are healthy deployments in the same model group
    2. there are fallbacks for the completion call
Nrr   r   r  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   rC  )
r  r  r  rP  r  r@  rA  r   _calculate_retry_afterr   )r  r  r  r   r   r  rB  r   s           r   r  "Router._time_to_sleep_before_retry  s    " &3+?1+D+.55'(1,481j!!gajj)&D&D zz111011 99'44"3'!1 ,,	G  44"3' ,,G r   c                   #    SSK Jn   UR                  SS5      nUc  [        S5      eUS   R                  S5      c  gUS   S   R                  SS5      nUS   S   R                  S	S5      nUS   R                  S
0 5      =(       d    0 n	U	R                  SS5      n
Ub  U
c  g[	        U
[
        5      (       a  [        U
5      n
U R                  U
S9nUc  gU R                  UR                  5       US9nUR                  SS5      nUR                  SS5      nUR                  R                  nUR                  R                  nUR                  SS5      nUR                  SS5      n[        U U
S9  Uc  Uc  Uc
  Uc  Uc  Uc  g[        U5      nUR                  SS5      n[        5       nUR!                  S5      n["        R$                  R&                  R)                  U
UUS9n/ nUR+                  U" UU[,        R.                  R&                  S95        ["        R0                  R&                  R)                  U
UUS9nUR+                  U" US[,        R.                  R&                  S95        U R2                  R5                  UUS9I Sh  vN   U$  N! [6         a8  n[8        R:                  " SR)                  [        U5      5      5         SnAgSnAff = f7f)z7
Track remaining tpm/rpm quota for model in model_list
r   )RedisPipelineIncrementOperationstandard_logging_objectNzstandard_logging_object is Noner   r   r  rH  r  r  r  r  received_model_nametpmrpmr  deployment_idtotal_tokens%H-%Mr  current_minuter   )r  increment_valuer   rr   )increment_listr  zOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})litellm.types.cachingrG  r1  r  r  r   r   get_deploymentget_router_model_infor  r   rL  rM  rD   r   rp   strftimerY   TPMr:  r  r   rx   r   RPMr   async_increment_cache_pipeliner  r   r  )r  r  completion_responser  r  rG  rH  deployment_namerH  r  r  deployment_infodeployment_model_inforL  rM  tpm_litellm_paramsrpm_litellm_paramstpm_model_inforpm_model_infor  rP  dtrS  tpm_keypipeline_operationsrpm_keyr  s                              r   r  %Router.deployment_callback_on_success  s2     	Jt	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# %%56zBFF!4 $$4599,KQr
^^D$/&"*C((RB #'"5"5r"5"B"*,0,F,F#2#=#=#?,7 -G -)
 *--eT:C)--eT:C *9)G)G)K)K&)8)G)G)K)K& &;%>%>ud%KN%:%>%>ud%KN B,0"$ K*2*2&.&.#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  NP# $**3#(4'OO11 *--33::. ;  $**3#()'OO11 jj??#6%5 @   
   	!''ahhF
 	sr   K5J K A&J &K'2J KB.J K	DJ JJ KJ 
K#.KKKKc                 0   SnUS   R                  S5      c  OoUS   S   R                  SS5      nUS   R                  S0 5      =(       d    0 nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      nUb  [	        U US9nU$ g)z
Tracks the number of successes for a deployment in the current minute (using in-memory cache)

Returns:
- key: str - The key used to increment the cache
- None: if no key is found
Nr   r   rH  r  r  rN  )r1  r  r   r   rD   )	r  r  r]  r  r  r  rH  r  r  s	            r   r	  *Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$$W>C(, C Jr   c                    [         R                  " S5         UR                  SS5      n[        USS5      nUR                  S0 5      nUR                  S0 5      n[        R
                  R                  R                  US9n	UR                  S	S5      n
SnU	b  [        R                  R                  U	S
9nU
b	  U
S:  a  U
nOUb	  US:  a  UnOU R                  n[        U[        5      (       a.  UR                  S5      nUc  g[        U US9  [        U UUUUS9nU$ [         R                  " S5        g! [         a  nUeSnAff = f)a(  
2 jobs:
- Tracks the number of failures for a deployment in the current minute (using in-memory cache)
- Puts the deployment in cooldown if it exceeds the allowed fails / minute

Returns:
- True if the deployment should be put in cooldown
- False if the deployment should not be put in cooldown
z1Router: Entering 'deployment_callback_on_failure'r  Nstatus_coder  r   r  )r  r   )rB  r   r  FrN  r  exception_statusr  r  time_to_cooldownzWRouter: Exiting 'deployment_callback_on_failure' without cooldown. No model_info found.)r   r  r1  rM  r   litellm_core_utilsexception_mapping_utils_get_response_headersutils&_get_retry_after_from_exception_headerr   r  r  rC   r:   r  )r  r  r]  r  r  r  ro  r   _model_infoexception_headersdeployment_cooldownheader_cooldown_time_to_cooldownrO  r  r  s                   r   r  %Router.deployment_callback_on_failure  ss     	##$WX9	

;5I&y-D $ZZ(8"=N(,,\2>K ' : : R R h h#, !i !
 #1"4"4_d"K"O ,")--"V"V%6 #W # #.3F!3K$7! ,A1E$3!$($6$6!+t,,/:t/D ( @,0"/ 3,0%5'0,%6 %++m  	G	s$   C+D4 D4 D4 4
E>E  Er]  c                 2  #    US   S   R                  SS5      nUS   S   R                  SS5      nUS   R                  S0 5      =(       d    0 nUR                  SS5      nUb  Uc  g[        U[        5      (       a  [        U5      n[	        U5      n	[        5       n
U
R                  S5      n[        R                  R                  R                  XUS	9nU R                  R                  US
U	[        R                  R                  S9I Sh  vN   g N7f)z#
Update RPM usage for a deployment
r   r   r  NrH  r  r  rQ  rR  rr   )r  r:  r  r   )r1  r  r   r   r   rp   rY  rY   r[  r:  r  r   async_increment_cacherx   r   )r  r  r]  r  r  r^  rH  r  r  r  re  rS  rh  s                r   r  +Router.async_deployment_callback_on_failure  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C  RB<VD

 "%%++22 3 
 jj..-%%	 / 
 	
 	
s   DDDD)r   rp  c                     [        U5      $ )ak  
Helper to return what the "metadata" field should be called in the request data

- New endpoints return `litellm_metadata`
- Old endpoints return `metadata`

Context:
- LiteLLM used `metadata` as an internal field for storing metadata
- OpenAI then started using this field for their metadata
- LiteLLM is now moving to using `litellm_metadata` for our metadata
)r   )r  r  s     r   '_get_metadata_variable_name_from_kwargs.Router._get_metadata_variable_name_from_kwargs0  s     6f==r   c                 .    SU;   a  SOSn[        U5      R                  [        U5      S.nUR                  5        H`  u  nnXSSS4;  a  XdU'   M  XS:X  d  M  [	        U[
        5      (       d  M2  0 XC'   X   R                  5        H  u  pxUS:w  d  M  XU   U'   M     Mb     [        U R                  5      S:  a  U R                  R                  S5        U R                  R                  U5        U R                  X   S'   U$ ! [         a  nUeS	nAff = f)
zs
When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
rp  r   )exception_typeexception_stringr  r  r      r   N)r   r|   r   r}  r  r  r  r   r  r   r  )	r  r  r  _metadata_varprevious_modelr!  r"  
metadata_k
metadata_vs	            r   r  Router.log_retry@  s   	&8F&B"
 
 #'q'"2"2$'FN 	 J8KLL()1%'Jq$,?,?46N1282G2M2M2O.
%)::<F1-j9 3P  4''(1,$$((+  ''77;7K7KF!"34M 	G	s+   AD D /D A0D 
DDDrO  r  c                     UnU R                   R                  X2SS9nUc  SnU R                   R                  X4SSS9  U$ US-  nU R                   R                  X4SS9  U$ )zF
Update deployment rpm for that minute

Returns:
- int: request count
T)r  r  
local_onlyrr   rz   )r  r:  r  r   )r  r:  r  )r   	get_cache	set_cache)r  rO  r  rh  request_counts        r   _update_usageRouter._update_usagee  s      

,,t - 
  MJJ  Tr !   QMJJ  T !  r   c                     U R                   c  gU R                    H"  n[        U[        5      (       d  M  SU;   d  M"    g   g)NFr   T)r   r  r  )r  r;  s     r   _has_default_fallbacksRouter._has_default_fallbacks  s9    >>!H(D))(? ' r   r  c                    UR                   (       a7  [        UR                   5      S:  a  UR                   S   R                  S:w  a  gUR                  SU R                  5      nUb6  SnU H)  n[        UR                  5       5      S   U:X  d  M%  Xa   n  O   Ub  gOU R                  5       (       a  g[        R                  " SR                  X5      5        g)z
Determines if a content policy error should be raised.

Only raised if a fallback is available.

Else, original response is returned.
r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})choicesr  finish_reasonr1  r   r  r8  r  r   r  r  )r  r   r  r  r   r  rU  s          r   r  )Router._should_raise_content_policy_error  s     H$4$4 5 9"004DD#)::&(E(E$
 
 $/#' 0		$Q'50+/;( 1
 $/ 0((**## H  O  O	

 r   c                     / n U R                  US9u  pC[        U[        5      (       a  / $  [	        XS9n/ nU H"  nUS   S   U;   a  M  UR                  U5        M$     Xc4$ ! [         a     NBf = f)Nrx  r  r  r  )#_common_checks_available_deploymentr  r  r  r9   r   )r  r   r  r   r  unhealthy_deploymentsr   r  s           r   _get_healthy_deploymentsRouter._get_healthy_deployments  s    !#	"&"J"J #K #A *D11	 2
 !:$(!
 %'*J,'-1FF#**:6	 + #44  		s   'A# #
A0/A0c                   #    / n U R                  US9u  pC[        U[        5      (       a  / U4$  [	        XS9I Sh  vN n[        U5      n/ nU H"  nUS   S   U;  d  M  UR                  U5        M$     Xs4$ ! [         a     NUf = f NL7f)z
Returns Tuple of:
- Tuple[List[Dict], List[Dict]]:
    1. healthy_deployments: list of healthy deployments
    2. all_deployments: list of all deployments
rx  r  Nr  r  )r  r  r  r  r7   setr   )	r  r   r  r   r  r  unhealthy_deployments_setr   r  s	            r   r  %Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D11+++ 2
 'F$('
 !
 %((=$>!$&*J,'-5NN#**:6 + #44  		!
s7   B)A: BB
 B"B:
BBBBc                     [         R                   H+  n[        U[        5      (       d  M  UR	                  U5        M-     g)z
Mimics 'async_routing_strategy_pre_call_checks'

Ensures consistent update rpm implementation for 'usage-based-routing-v2'

Returns:
- None

Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
N)r   r+  r  r   r  )r  r  r  s      r   r  'Router.routing_strategy_pre_call_checks  s/     !**I)\22((4 +r   r0  c           
        #    [         R                   H4  n[        U[        5      (       d  M   UR	                  X5      I Sh  vN   M6     g N	! [         R
                   a  nUb  [        R                  " UR                  U[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U[        R                  " 5       4S9R                  5         [!        U UR"                  UUS   S   U R$                  S9  UeSnAf[&         a  nUb  [        R                  " UR                  U[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U[        R                  " 5       4S9R                  5         UeSnAff = f7f)  
For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

-> makes the calls concurrency-safe, when rpm limits are set for a deployment

Returns:
- None

Raises:
- Rate Limit Exception - If the deployment is over it's tpm/rpm limits
Nr  traceback_exceptionr  targetrI  r  r  rn  )r   r+  r  r   async_pre_call_checkr,  r  r  async_failure_handlerr"  r#  r  	threadingThreadfailure_handlerstartr:   rm  r   r  )r  r  r  r0  r  r  s         r   rk  -Router.async_routing_strategy_pre_call_checks  s|    " !**I)\22(#88VVV + W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsH   'GAAAGAF?"B9DF?(BF::F??Gr  c           
        #    Un[         R                   H6  n[        U[        5      (       d  M   UR	                  UUUUUS9I Sh  vN nM8     U$  N
! [
         a  n	Ub  [        R                  " UR                  U	[        R                  " 5       [        R                  " 5       S95        [        R                  " UR                  U	[        R                  " 5       4S9R                  5         U	eSn	A	ff = f7f)r  r   r   r  r  r  Nr  r  )r   r+  r  r   async_filter_deploymentsr  r  r  r  r"  r#  r  r  r  r  r  )
r  r   r   r  r  r  r0  returned_healthy_deploymentsr  r  s
             r   !async_callback_filter_deployments(Router.async_callback_filter_deployments1  s     ( (;$ **I)\22'@@"'0L%-+9-= A   1 +8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   )C7AAAC7A
C4BC//C44C7r   c                    U/nUR                  5        GH   u  pE[        U[        5      (       a  UR                  U5        OU[        U[        5      (       a&  UR                  [
        R                  " U5      5        OUR                  [        U5      5        [        U[        5      (       a  UR                  U5        M  [        U[        5      (       a'  UR                  [
        R                  " U5      5        M  UR                  [        U5      5        GM     SR                  U5      n[        R                  " UR                  5       5      nUR                  5       $ )z
Helper function to consistently generate the same id for a deployment

- create a string from all the litellm params
- hash
- use hash as id
r  )r}  r  r   r   r  jsondumpsjoinhashlibsha256encode	hexdigest)r  rH  r   partsr!  r"  
concat_strhash_objects           r   r  Router._generate_model_idd  s     "((*DA!S!!QAt$$TZZ]+SV$!S!!QAt$$TZZ]+SV$ + WWU^
nnZ%6%6%89$$&&r   r_  _model_name_litellm_paramsrv  c                     [        S0 UD6n[        S0 UDUUUS.D6n[        R                  R	                  5        H2  nUR
                  R                  U5      c  M!  UR
                  U   XG'   M4     UR                  R                  nUb  [        R                  " X0S9  UR
                  R                  nUR
                  R                  b  UR
                  R                  S-   U-   n[        R                  " X$0S9  U R                  US9SLa4  [        R                  " SUR                    SUR                  S	    35        gU R#                  US9nUR%                  SS
9n	U R'                  XR                  R                  S9  U$ ! [(         a7  n
U R*                  (       a  [        R,                  " SU
 S35         Sn
A
gU
eSn
A
ff = f)a.  
Create a deployment object and add it to the model list

If the deployment is not active for the current environment, it is ignored

Returns:
- Deployment: The deployment object
- None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
r  N)
model_costr	  r  TzIgnoring deployment z% as it is not active for environment supported_environmentsr   r   r  zError creating deployment: 1, ignoring and continuing with other deployments.r{   )rT   rR   ra   model_fieldsr8  r   r1  r  r  r   register_modelr   r/  $deployment_is_active_for_environmentr   warningr  _add_deploymentto_json _add_model_to_list_and_index_mapr  r   r  )r  r_  r  r  rv  r   r  fieldr  r   r  s              r   _create_deploymentRouter._create_deployment  s    ;	-;-No-NN# !&-&	J 4@@EEG,,007C)3)B)B5)IK& H
 ",,//H#&&   %3399K((<<H--AACG+U  "" 99Z9P &--*:+@+@*AAfgqg|g|  ~V  hW  gX  Y ---DJ&&D&9E11&;&;&>&> 2   	..%//1!4ef 	s,   AF C&F AF 
G+GGGc                 F    UR                   R                  S5      (       a  gg)z|
Check if the deployment is an auto-router deployment.

Returns True if the litellm_params model starts with "auto_router/"
zauto_router/TF)r   
startswith)r  r   s     r   _is_auto_router_deployment!Router._is_auto_router_deployment  s!     **>::r   c           	         SSK Jn  UR                  R                  nUR                  R                  nUc  Uc  [        S5      eUR                  R                  nUc  [        S5      eUR                  R                  nUc  [        S5      eU" UR                  UUUUU S9nUR                  U R                  ;   a  [        SUR                   S	35      eXpR                  UR                  '   g)
zy
Initialize the auto-router deployment.

This will initialize the auto-router and add it to the auto-routers dictionary.
r   )ru   Nzzauto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_paramszfauto_router_default_model is required for auto-router deployments. Please set it in the litellm_paramszhauto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params)r  auto_router_config_pathauto_router_configdefault_modelembedding_modelr  zAuto-router deployment z3 already exists. Please use a different model name.)
/litellm.router_strategy.auto_router.auto_routerru   r   r  r  r  auto_router_default_modelauto_router_embedding_modelr  r   )r  r  ru   r  r  r  r  autor_routers           r   init_auto_router_deployment"Router.init_auto_router_deployment  s    	O %%== 	  -7,E,E,X,X"*/A/I M  %%?? 	  x  %%AA 	 "z  $.!,,$;1'+$($
   D$5$55)**?*?)@@st  4@*//0r   c           	      b   UR                   b   SUR                   ;  d  UR                   S   c  g[        SS9nUc  [        S5      eU[        ;  a  [        S[         SU 35      eUR                   S    H%  nU[        ;  d  M  [        S[         SU S	U 35      e   X!R                   S   ;   a  gg
)a  
Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
    - development
    - staging
    - production

Raises:
- ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
- ValueError: If supported_environments is not set in model_info or not one of the valid values
r  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r  r   r  rL   )r  r  litellm_environment_envs       r   r  +Router.deployment_is_active_for_environment	  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BCD55 <=W<XXfgkfll}  I  ~J  K  D "7"78P"QQr   c           	      2   [         R                  " U5      n/ U l        0 U l        0 U l        U GH  nUR                  S5      nUR                  S5      n[        U[        5      (       aU  UR                  5        HA  u  pg[        U[        5      (       d  M  UR                  S5      (       d  M4  [        U5      XV'   MC     UR                  S0 5      nSU;  a  U R                  XE5      n	XS'   UR                  SS 5      b=  [        US   [        5      (       a%  US    H  n
XS'   U R                  UUUUS9  M     GM  U R                  UUUUS9  GM"     [         R"                  " SU R%                  5        35        U Vs1 s H  oS   iM	     snU l        g s  snf )	Nr  r   os.environ/r  r  organization)r_  r  r  rv  z
Initialized Model List )r  r  r   r   r   r  r  r  r}  r   r  ro   r  r1  r  r  r   r  get_model_namesr   )r  r   original_model_listr   r  r  r!  r"  rv  _idorgr  s               r   r   Router.set_model_list1  s   "mmJ702-02- )E))L1K#ii(89O/400+113DA!!S))all=.I.I-7]* 4 !&		, ;K ;&--kK$'D!"">48D/J J +>:C69N3++(-$/(7$/	 ,  ; ''$) +$3 +	 ( 9 )F 	##'(<(<(>'?@	
 6@@ZlOZ@@s   =Fc                    SS K nUR                  R                  nSnSU;   a*  UR                  S5      S   nU[        R
                  ;   a  SnU(       a	  UnS nS nS n	Oj[        R                  " UR                  R                  UR                  R                  SS 5      S9u  nnnn	U[        R                  ;  a  [        SU 35      eU R                  R                  UR                  R                  5        UR                  R                  c)  [        USS 5      b  [        US5      UR                  l        UR                  R                  c)  [        US	S 5      b  [        US	5      UR                  l        S
UR                  ;   a}  U R                   R#                  UR                  UR%                  SS95        UR&                  R(                  (       a/  U R*                  R                  UR&                  R(                  5        UR&                  R                  S5      n
UR&                  R                  S5      nU
bZ  UbW  S
U;   aQ  XR,                  ;  a  [/        5       U R,                  U
'   U R,                  U
   R#                  XR%                  SS95        UR                  R                  S/ 5      =(       d    / nU Hq  nUR                  S0 5      nS HV  nX;   d  M
  X   R1                  S5      (       d  M$  X   R3                  SS5      nUR4                  R                  US5      X'   MX     Ms     Ub%  U R7                  UUUR                  R                  S9  U R9                  UR                  S9(       a  U R;                  US9  U$ )Nr   Fr	  Tr/  r   r/  zUnsupported provider - rM  rL  r   r   team_idteam_public_model_namedataSources
parameters)rv  r  r  r  )r  r/  r   )r   r  )osr   r   r  r   r  rm   r1  provider_listr  r   r   rM  rM  rL  r  r   add_patternr  r  r  r   r   rs   r  rw  environ'_initialize_deployment_for_pass_throughr  r  )r  r  r  r  is_prompt_management_modelr  r"  r/  r  r  _team_id_team_public_model_namedata_sourcesdata_sourcer   	param_keyenv_names                    r   r  Router._add_deploymentc  sN    #1177%*"-"/"5"5c":1"="g&W&WW-1*% #F"&"OH (( //55$.$=$=$A$A)4%# #'*?*??"9:M9N OPP 	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU((,,Y7","7"7";";<T"U '3..8886H6J))(3%%h/;;');););)N
 "0044]BGM2'K __\26F0	&6+<+G+G+V+V%088KH(*

x(DF%	 1 ( *88%$7 //55 9  ***:S:S*T,,
,Cr   c                    UR                   R                  SL Gak  SSKJn  UR                   R                  b+  [
        R                  " UR                   R                  5      nO0 nUS:X  a  UR                  S5      =(       d    UR                   R                  nUR                  S5      =(       d    UR                   R                  nUR                  S5      =(       d    UR                   R                  nUb  Uc  [        S	5      eUR                  UUUS
9  OmUR                  S5      =(       d    UR                   R                  n	UR                  S5      =(       d    UR                   R                  n
UR                  UU	U
S9   g)a  
Optional: Initialize deployment for pass-through endpoints if `deployment.litellm_params.use_in_pass_through` is True

Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
Tr   )passthrough_endpoint_routerN	vertex_aivertex_projectvertex_locationvertex_credentialsz]vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints.)
project_idlocationr
  r  r  )r/  r  r  )r   use_in_pass_through>litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsr  litellm_credential_namer!   get_credential_valuesr1  r  r	  r
  r  add_vertex_credentialsr  r  set_pass_through_credentials)r  r  r/  r   r  credential_valuesr  r	  r
  r  r  s              r   r  .Router._initialize_deployment_for_pass_through  s    $$88D@ ((@@L$6$L$L--EE%! %'!"k1%))*:; @!00?? 
 &))*;< A!00@@  
 &))*>? D!00CC #
 ")_-D$w  ,BB-,'9 C  &))*5 :!0099 
 &)))4 9!0088  ,HH(;%# I 
 r   c                 :   UR                   R                  nU(       a  U R                  U5      (       a  gUR                  SS9nU R	                  US9  U R                  X1R                   R                  S9  U R                  R                  UR                  5        U$ )z
Parameters:
- deployment: Deployment - the deployment to be added to the Router

Returns:
- The added deployment
- OR None (if deployment already exists)
NTr   r  r  )	r  r  r  r  r  r  r   addr  )r  r  _deployment_model_id_deployments       r   add_deploymentRouter.add_deployment  s      *4477D$5$56J$K$K !((d(;
3 	--(=(=(@(@ 	. 	
 	Z223r   r  removal_idxc                    U R                   R                  5        H  u  p4XB:  d  M  US-
  U R                   U'   M      XR                   ;   a  U R                   U	 [        U R                  R                  5       5       H~  u  pVX&;   a  UR	                  U5        / nU H/  nXB:  a  UR                  US-
  5        M  UR                  U5        M1     [        U5      S:  a  XpR                  U'   Mq  U R                  U	 M     g)z
Helper method to update deployment indices after a deployment has been removed from model_list.

Parameters:
- model_id: str - the id of the deployment that was removed
- removal_idx: int - the index where the deployment was removed from model_list
rr   r   N)r   r}  r  r   r  r   r  )r  r  r  rO  r  r  indicesupdated_indicess           r   (_update_deployment_indices_after_removal/Router._update_deployment_indices_after_removal/  s     #'"G"G"M"M"OM GJQw55mD #P <<<55h? $((M(M(S(S(U#VJ%{+ !O$#**373#**3/	  ?#a'DS55jA99*E# $Wr   c                    [        U R                  5      nU R                  R                  U5        Ub  X0R                  U'   O6UR	                  S0 5      R	                  S5      b  X0R                  US   S   '   UR	                  S5      nU(       a=  X@R
                  ;  a  / U R
                  U'   U R
                  U   R                  U5        gg)z
Helper method to add a model to the model_list and update both indices.

Parameters:
- model: dict - the model to add to the list
- model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
Nr  r  r  )r  r   r   r   r1  r   )r  r   r  r  r  s        r   r  'Router._add_model_to_list_and_index_mapU  s     $//"u% >A11(;YY|R(,,T2>OR11%2Ed2KL YY|,
!F!FFDF55jA11*=DDSI r   c                     UR                   R                  =(       d    SnU R                  US9nUbu  UR                  UR                  :X  a  gSnUR                   R                  nU R                  nXV;   a1  Xe   nUb*  U R
                  R                  U5        U R                  XTS9  U R                  US9  U$ ! [         a7  nU R                  (       a  [        R                  " SU S35         SnAgUeSnAff = f)z
Add or update deployment
Parameters:
- deployment: Deployment - the deployment to be added to the Router

Returns:
- The added/updated deployment
r  rI  Nr  r  r  zError upserting deployment: r  )r  r  rW  r   r   r   r  r  r  r  r   r   r  )r  r  r  _deployment_on_routerr  rO  deployment_fast_mappingr  s           r   r  Router.upsert_deploymento  s   &	#-#8#8#;#;#Ar :>:M:M- ;N ;! %0,,0E0T0TT .2 * 5 5 8 8*.*O*O' ;"9"HK".++K8EE%2 F 
 :6 	..%++21#5fg 	s%   AB9 A*B9 9
C:+C53C55C:r  c                     SnXR                   ;   a  U R                   U   n Ub,  U R                  R                  U5      nU R                  XS9  U$ g! [         a     gf = f)z
Parameters:
- id: str - the id of the deployment to be deleted

Returns:
- The deleted deployment
- OR None (if deleted deployment not found)
Nr$  )r   r   r  r  r  )r  r  deployment_idxrU  s       r   delete_deploymentRouter.delete_deployment  sv     666!BB2FN	)**>:== >   		s   .A 
AAc                    XR                   ;   ax  U R                   U   nU R                  U   n[        U[        5      (       a  [	        S0 UD6$ [        U[        5      (       a  U$ [        SR                  [        U5      5      5      eg)zT
Returns -> Deployment or None

Raise Exception -> if model found in invalid format
zModel invalid format - {}Nr{   )r   r   r  r  rR   r  r  r   )r  r  r  r   s       r   rW  Router.get_deployment  sy     <<<77ACOOC(E%&&!*E**E:.. ; B B4; OPPr   c                     U R                  US9nUc  g[        S0 UR                  R                  SS9D6R                  SS9$ )z5
Returns -> dict of credentials for a given model id
rI  NTr   r{   )rW  rP   r   r  )r  r  r  s      r   get_deployment_credentials!Router.get_deployment_credentials  sU     ((((;
& 
''222E

*$*
'	(r   model_group_namec                 &   XR                   ;   a  U R                   U   nU(       al  U R                  US      n[        U[        5      (       a  [	        S0 UD6$ [        U[        5      (       a  U$ [        SR                  [        U5      5      5      eg)z
Returns -> Deployment or None

Raise Exception -> if model found in invalid format

Optimized with O(1) index lookup instead of O(n) linear scan.
r   zModel Name invalid - {}Nr{   )r   r   r  r  rR   r  r  r   )r  r1  r  r   s       r   "get_deployment_by_model_group_name)Router.get_deployment_by_model_group_name  s     DDD;;<LMG
3eT**%...z22 L#$=$D$DT%[$QRRr   rK  c                     g rE  r{   r  r  rK  r  s       r   rX  Router.get_router_model_info       	r   c                     g rE  r{   r6  s       r   rX  r7    r8  r   c                 .   Ub!  U R                  US9nUb  UR                  SS9nUc  [        S5      eUR                  S0 5      R                  SS5      nUc"  UR                  S0 5      R                  SS5      nUn[        R
                  " UR                  S0 5      R                  S	S
5      [        S0 UR                  S0 5      D6S9u  px  n	US:X  a  Uc  [        R                  " SU S35        OUS:w  a  UnU R                  R                  U5      n
SU;   at  U
bq  U
 Hk  n UR                  S0 5      R                  S5      UR                  S0 5      R                  S5      :X  a#  UR                  S0 5      R                  S	5      n  OMm     Uc  UnUR                  SR                  U5      5      (       d  SR                  X5      nOUn[        R                  " US9nUR                  S0 5      nUR                  U5        U$ ! [         a     M  f = f)a  
For a given model id, return the model info (max tokens, input cost, output cost, etc.).

Augment litellm info with additional params set in `model_info`.

For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

Returns
- ModelInfo - If found -> typed dict with max tokens, input cost, etc.

Raises:
- ValueError -> If model is not mapped yet
NrI  Tr   zDeployment not foundr  
base_modelr   r   r  r   r   azurez Could not identify azure model 'z'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   r  z{}/z{}/{}rx  r{   )rW  r  r  r1  r   rm   rT   r   rT  r   router  r  r  get_model_infor   )r  r  rK  r  r  r;  r   r"  r/  r  potential_modelspotential_modelmodel_info_namer  user_model_infos                  r   rX  r7    sA   & >--r-:K&(333F
344  ^^L"599,M
#(8"=AA,PTUJ -4,D,D..!126::7BG)QJNN;KR,PQ-
)Q ')j.@!''26(  ;Z  [ !G+E#22889LMe| 0 <'7O	*..|R@DD '^^L"=AA$GH %4$7$78H"$M$Q$Q '%E "H (8 = E-@ ABB%nn-@HO#O++/B
 %..r:/*+ % s   $A%H
HHc                 ^    XR                   ;   a  U R                   U   nU R                  U   $ g)z
For a given model id, return the model info

Returns
- dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
- None: could not find deployment in list

Optimized with O(1) index lookup instead of O(n) linear scan.
N)r   r   )r  r  r  s      r   r?  Router.get_model_infoN  s2     66677;C??3''r   c                 P    U R                  US9nUc  gUS   nU R                  US9$ )zD
Return list of all models in the same model group as that model id
)r  Nr  r  )r?  r
  )r  r  r  r  s       r   get_model_groupRouter.get_model_group^  s>    
 ((B(/
-
""j"99r   c           	         SSK Jn  SnSnSn [        R                  R	                  U5      n [        R                  " US9n UbN  UR	                  SS5      nUb9  [        R                  " US9nUb"  U=(       d    0 nU" [        [        U5      U5      nUb8  Ub5  [        [        U" [        [        U5      R                  5       U5      5      nU$ Ub  UnU$ ! [
         a     Nf = f! [
         a     Nf = f! [
         a     Nnf = f)z
For a given model id, return the model info

1. Check if model_id is in model info
2. If not, check if litellm model name is in model info
3. If not, return None
r   )_update_dictionaryNrx  r;  )litellm.utilsrJ  r   r  r1  r  r?  r   r  rd   r  )	r  r  r  rJ  r  custom_model_infolitellm_model_name_model_infor;  base_model_infos	            r   get_deployment_model_info Router.get_deployment_model_infoj  s6    	5*.
,0=A%	 ' 2 2 6 6x @	,3,B,B,T)
	 ,.22<F
)&-&<&<:&NO&2,=,C),> 7--) (-J-V"<=BBD%J  +66JI  		
  		"  		s5   C C& AC6 
C#"C#&
C32C36
DDuser_facing_model_group_namec                 
   SnSnSnSnU R                  US9nUc  gU GHY  nSn	SU;   a  US   U:X  a  Sn	O$SU;   a  U R                  R                  U5      b  Sn	U	(       d  ME  [        S0 US   D6n
U
R                  nUR                  S0 5      nUR                  S0 5      nSnUc  UR                  SS5      nUc  UR                  SS5      nUc  UR                  SS5      nSnUc  UR                  S	S5      nUc  UR                  S	S5      nUc  UR                  S	S5      n UR                  S
S5      nUb  U R                  XR                  S9nOSn Su  nn [        R                  " U
R                  U
R                  S9u  nn  nUcP  [        R$                  " UUS9nUc  / nUR                  S0 5      nUR                  SS5      n['        USSSSSUUUSS9
nUc  [)        S0 UU/S.UED6nGOiUUR*                  ;  a  UR*                  R-                  U5        UR                  SS5      b0  US   b*  UR.                  b  US   UR.                  :  a
  US   Ul        UR                  SS5      b0  US   b*  UR0                  b  US   UR0                  :  a
  US   Ul        UR                  SS5      b*  UR2                  b  US   UR2                  :  a
  US   Ul        UR                  SS5      b*  UR4                  b  US   UR4                  :  a
  US   Ul        UR                  SS5       b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl        UR                  SS5      b  US   SL a  SUl         UR                  SS5      b  US   b
  US   Ul!        UR                  SS5      b  Uc  UR                  S5      nUR                  S	S5      b  Uc  UR                  S	5      nUb	  Uc  SnXM-  nUc  GMP  Uc  SnX^-  nGM\     Ub  Ub  XCl"        Ub  XSl#        Ub  UUl        U$ ! [         a    Sn GNVf = f! [        R                  R                   a9  n[        R                  " SR!                  [#        U5      5      5         SnAGNxSnAff = f) z
For a given model group name, return the combined model info

Returns:
- ModelGroupInfo if able to construct a model group
- None if error constructing model group info
Nr  Fr  Tr   r  rL  rM  r  )r  r  )r  r  r  z.litellm.router.py::get_model_group_info() - {}moder   r   )
r  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providerrS  supported_openai_paramssupports_system_messages)rH  	providersrU  rV  rW  rX  "supports_parallel_function_callingsupports_visionsupports_function_callingsupports_web_searchsupports_url_contextsupports_reasoningrZ  r{   )$r
  r   r>  rT   #configurable_clientside_auth_paramsr1  rO  r   r  r   rm   r/  
exceptionsBadRequestErrorr   rT  r  r   get_supported_openai_paramsModelMapInforV   r\  r   rU  rV  rW  rX  r]  r^  r_  r`  ra  rb  rZ  rL  rM  )r  rH  rQ  model_group_info	total_tpm	total_rpmrc  r   r   is_matchr   model_litellm_paramsmodel_info_dict_deployment_tpm_deployment_rpmr  r  r  r  r  r  rZ  db_model_inforS  s                           r   _set_model_group_infoRouter._set_model_group_info  s0    6:#'	#'	SW+((K(@
EH%%*=*L%''--k:F+Fe4D.EFN BB 0
 $)99-=r#B #iib9O .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B	"*..tT:'!%!?!?!)6J6J "@ "J "&J
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+ !&		, ;$((8)##%)&*)**+%1,C-1
  '#1 $'C&2^ %$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#8$?K"#89TA;?$8NN#94@L"#9:dB<@$9 NN#7>J"#78D@:>$7NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I,	*$ !I,	K  L '$'0$$'0$ 3>7 !D  A  "!
" %%55 %++DKKCPQFS s0   $.RR.R0R-,R-0T.TTc                     XR                   ;   a\  U R                   U   n[        U[        5      (       a  UnO%[        U[        5      (       a  US   SL a  gUS   nOgU R	                  UUS9$ U R	                  XS9$ )z
For a given model group name, return the combined model info

Returns:
- ModelGroupInfo if able to construct a model group
- None if error constructing model group info or hidden model group
hiddenTNr   )rH  rQ  )r   r  r   r  rq  )r  rH  rU  _router_model_groups       r   get_model_group_infoRouter.get_model_group_info  s     000))+6D$$$&*#D$''>T)*.w-'--/-8 .   ))# * 
 	
r   c           	      *  #    [        5       nUR                  S5      n/ n/ nU R                  US9nUc  gU H  nUR                  S0 5      R                  S5      nUS   R                  S5      n	Ub  U	c  M@  UR	                  [
        R                  R                  R                  UU	US	95        UR	                  [
        R                  R                  R                  UU	US	95        M     XE-   n
U R                  R                  U
S
9I Sh  vN nUc  gUS[        U5       nU[        U5      S nSnUb)  U H#  n[        U[        5      (       d  M  Uc  SnX-  nM%     SnUb*  U H$  n[        U[        5      (       d  M  Uc  SnUU-  nM&     UU4$  N7f)z
Returns current tpm/rpm usage for model group

Parameters:
- model_group: str - the received model name from the user (can be a wildcard route).

Returns:
- usage: Tuple[tpm, rpm]
rQ  r  Nr  r  r  r   r   )r  r   rS  )r8  r   )rp   rY  r
  r1  r   rY   rZ  r:  r  r[  r   async_batch_get_cacher  r  r   )r  rH  re  rS  tpm_keysrpm_keysr   r   r  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    r   get_model_group_usageRouter.get_model_group_usage  s     
 ! ((K(@
E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1   * !) 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%#a%% ($%	NI	 $ $(	%#a%% ($%	NI	 $
 )##3#
s   D	FFAF*F?F)maxsizec                 $    U R                  U5      $ )z
Cached version of get_model_group_info, uses @lru_cache wrapper

This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
)rv  )r  rH  s     r   _cached_get_model_group_info#Router._cached_get_model_group_info  s     ((55r   c                 Z  #    U R                  U5      nUb  UR                  b  UR                  nOS nUb  UR                  b  UR                  nOS nUc  Uc  0 $ U R                  U5      I S h  vN u  pV0 nUb  UU=(       d    S-
  US'   X7S'   Ub  UU=(       d    S-
  US'   XGS'   U$  N:7f)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r  rL  rM  r  )r  rH  rh  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           r   get_remaining_model_group_usage&Router.get_remaining_model_group_usage  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P  <E q=M89 9B45 >G q?M:; ;D67 $Qs   A,B+.B)/;B+c                   #    [        U[        5      (       a  [        US5      (       a  [        UR                  [        5      (       a  UR                  R                  S0 5        UUR                  S   S'   UR                  S   nSU;  aD  SU;  a>  Ub;  U R                  U5      I Sh  vN nUR                  5        H  u  pVUc  M
  XcU'   M     U$  N(7f)z
Add the most accurate rate limit headers for a given model response.

## TODO: add model group rate limit headers
# - if healthy_deployments > 1, return model group rate limit headers
# - else return the model's rate limit headers
r  r  zx-litellm-model-groupr  r  N)r  r   rP  r  r  r   r  r}  )r  r  rH  r  remaining_usageheaderr:  s          r   r(  Router.set_response_headers  s      x++"233822D99##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%<MF(5:62 &= #s   B&C(C)CCc                    U R                   R                  5         [        U5       H[  u  p#UR                  S5      nU(       d  M  X@R                   ;  a  / U R                   U'   U R                   U   R	                  U5        M]     g)z
Build model_name -> deployment indices mapping for O(1) lookups.

This index allows us to find all deployments for a given model_name in O(1) time
instead of O(n) linear scan through the entire model_list.
r  N)r   clearr  r1  r   )r  r   r  r   r  s        r   _build_model_name_indexRouter._build_model_name_indexA  so     	--335#J/JC<0Jz%J%JJHJD99*E55jAHHM 0r   c                 2   / U l         [        U5       H  u  p#UR                  S0 5      nUR                  S5      nUcG  UR                  SS5      nUR                  S0 5      nU R                  Xg5      nSU;  a  0 US'   XSS   S'   U R	                  X5S9  M     g)z
Build model index from model list to enable O(1) lookups immediately.
This is called during initialization to avoid the race condition where
requests arrive before model_id_to_deployment_index_map is populated.
r  r  Nr  r  r   r  )r   r  r1  r  r  )r  r   r  r   r  r  r  r   s           r   '_build_model_id_to_deployment_index_map.Router._build_model_id_to_deployment_index_mapQ  s     !*-HA<4J!~~d+H "YY|R8
!&+;R!@22:Nu,*,E,',4l#D)111Q .r   exclude_team_modelsc                 "   / nUb  XR                   ;   as  U R                   U   nU H^  nU R                  U   nSU;   d  M  SUS   ;   d  M%  U(       a  US   R                  S5      (       a  MG  UR                  US   S   5        M`     U$ U R                  R                  5        Hg  nU R                  U   nU R                  U   nSU;   d  M)  SUS   ;   d  M4  U(       a  US   R                  S5      (       a  MV  UR                  U5        Mi     U$ )z
if 'model_name' is none, returns all.

Returns list of model id's.

Optimized with O(1) or O(k) index lookup when model_name provided,
instead of O(n) linear scan.
r  r  r  )r   r   r1  r   r   r8  )r  r  r  idsr  r  r   r  s           r   get_model_idsRouter.get_model_idsj  s    !BBB??
K"C OOC0E#u,|9L1L.53F3J3J93U3U$

5#6t#<= #" 
 !AAFFH;;HE,5(TU<5H-H*u\/B/F/Fy/Q/Q JJx( I 
r   candidate_idc                     XR                   ;   $ )ar  
O(1) membership check for a deployment ID without allocating large lists.

Note: Call sites may pass a variable named `model` when it actually
contains a deployment ID. This helper expects the deployment ID string.

Uses the existing `model_id_to_deployment_index_map` which is kept
in sync by `_build_model_id_to_deployment_index_map` and model-list
mutation helpers.
)r   )r  r  s     r   r  Router.has_model_id  s     DDDDr   team_model_namer  c                     U R                  XS9nU(       d  gU H;  nUR                  S0 5      R                  S5      U:X  d  M*  UR                  S5      s  $    g)z
Map a team model name to a team-specific model name.

Returns:
- deployment id: str - the deployment id of the team-specific model
- None: if no team-specific model name is found
r  r  Nr  r  r  )r
  r1  )r  r  r  r  r   s        r   map_team_modelRouter.map_team_model  sY     $$$QEyyr*..y9WDyy.. 
 r   c                     Ub0  US   R                  S5      U:X  a  XS   R                  S5      :X  a  gUb
  US   U:X  a  gg)zE
Get the team-specific model name if team_id matches the deployment.
r  r  r  Tr  Fr1  )r  r  r   r  s       r   should_include_deployment Router.should_include_deployment  sT     l#''	2g=L1556NOO#l(;z(Ir   model_aliasc                    / nXR                   ;   a{  U R                   U   nU Hd  nU R                  U   nU R                  XUS9(       d  M)  Ub'  UR                  5       nX(S'   UR	                  U5        MS  UR	                  U5        Mf     U$ Ubp  [        U R                  5       HW  u  pgU R                  XUS9(       d  M  Ub'  UR                  5       nX(S'   UR	                  U5        MF  UR	                  U5        MY     U$ )z
Return all deployments of a model name

Used for accurate 'get_model_list'.

if team_id specified, only return team-specific models

Optimized with O(1) index lookup instead of O(n) linear scan.
)r  r   r  r  )r   r   r  r  r   r  )	r  r  r  r  returned_modelsr  r  r   alias_models	            r   _get_all_deploymentsRouter._get_all_deployments  s    68 >>>;;JGG ,11) 2   #. ',jjl4?L1'..{;'..u5 : !   (8
11) 2   #.&+jjl4?L1'..{;'..u5 9 r   c                 4   U R                  5       =(       d    / n/ nU Hv  nUR                  S5      nU R                  U5      (       a+  U R                  XAS9nU(       a  UR	                  U5        MS  MU  UR	                  UR                  SS5      5        Mx     U$ )u   
Returns all possible model names for the router, including models defined via model_group_alias.

If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models)
will yield their team public name.
r  )r  r  r  r  )r
  r1  _is_team_specific_model_get_team_specific_modelr   )r  r  deploymentsr   r  r  r  s          r   r  Router.get_model_names  s     ))+1r%J#5J++J77"&"?"?) #@ # #&&7 # "":>>,#CD & r   c                     UR                  S5      =(       d    0 nUc  gX#R                  S5      :X  a  UR                  S5      $ g)az  
Get the team-specific model name if team_id matches the deployment.

Args:
    deployment: DeploymentTypedDict - The model deployment
    team_id: Optional[str] - If passed, will return router models set with a `team_id` matching the passed `team_id`.

Returns:
    str: The `team_public_model_name` if team_id matches
    None: If team_id doesn't match or no team info exists
r  Nr  r  r  )r  r  r  r  s       r   r  Router._get_team_specific_model	  sB     &0^^L%A%GR
nnY//>>":;;r   r  c                 H    [        U=(       a    UR                  S5      5      $ )z
Check if model info contains team-specific configuration.

Args:
    model_info: Model information dictionary

Returns:
    bool: True if model has team-specific configuration
r  )r  r1  )r  r  s     r   r  Router._is_team_specific_model  s     J<:>>)#<==r   c                 4   / nU R                   R                  5        Hw  u  p4Ub  X1:w  a  M  [        U[        5      (       a  UnO2[        U[        5      (       a  [        S0 UD6nUS   SL a  MQ  US   nOMY  UR                  U R                  XSS95        My     U$ )zt
Helper function to get model list from model alias.

Used by `.get_model_list` to get model list from model alias.
rt  Tr   )r  r  r{   )r   r}  r  r   r  r[   r  r  )r  r  r  r  model_value_router_model_name_model_values          r   get_model_list_from_model_alias&Router.get_model_list_from_model_alias*  s     68(,(>(>(D(D(F$K%+*C+s++*5"K..8G;G)T1)5g)>&""))1 *  )G( r   c                    / nUb  UR                  U R                  XS95        UR                  U R                  US95        [        U5      S:X  a  U R                  R                  U5      =(       d    / nUbG  X R                  ;   a8  U R                  U   R                  U5      =(       d    / nUR                  U5        Ub,  Ub)  U H#  n[        S0 UD6nXS'   UR                  U5        M%     Uc  X0R                  -  nU$ )zk
Includes router model_group_alias'es as well

if team_id specified, returns matching team-specific models
r  r  r   r  r{   )
r  r  r  r  r   r>  r   rS   r   r   )r  r  r  r  potential_wildcard_models#potential_team_only_wildcard_modelsr  deployment_typed_dicts           r   r
  Router.get_model_listI  s    68!""))Z)Q 	00J0G	
 1$(,(;(;(A(A*(M(SQS% "w2K2K'K--g6<<ZHNB 4 *001TU%*C*O2A,?,D!,D):D,7#**+@A 3
 .Or   model_access_groupc                 X   SSK Jn  U" [        5      nU R                  XS9nU(       a  U H{  nUR	                  S5      nU(       d  M  UR	                  S/ 5      =(       d    /  H?  n	Ub!  X:X  a  US   nXY   R                  U5        M%  M'  US   nXY   R                  U5        MA     M}     U$ )a  
If model_name is provided, only return access groups for that model.

Parameters:
- model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
- model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
- team_id: Optional[str] - the team id, to resolve team-specific models
r   r   r  r  r   r  )collectionsr   r  r
  r1  r   )
r  r  r  r  r   r   r   r  rv  groups
             r   get_model_access_groupsRouter.get_model_access_groupss  s     	,#D)((J(P
eeL1;!,"!E!K!K-9$:-.|_
 - 4 ; ;J G  ; *+<J)077
C "L   r   c                     U R                  US9n[        U5      S:X  a  gUR                  U/ 5      nU H  nU R                  R	                  US9c  M    g   g)z7
Return True if model access group is a wildcard route
)r  r   Fr  T)r  r  r1  r   r>  )r  r  r   r  r   s        r   )_is_model_access_group_for_wildcard_route0Router._is_model_access_group_for_wildcard_route  sm     441 5 
 }"""#5r:E""(((7C 
 r   c                     [        U 5      n0 n/ SQnU HN  nXA;   a  X   X$'   US:X  d  M  U R                  S:X  d  M(  U R                  R                  R	                  5       X$'   MP     U$ )z
Get router settings method, returns a dictionary of the settings and their values.
For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )varsr   r@  r8  r  )r  	_all_vars_settings_to_returnvars_to_includevars        r   get_settingsRouter.get_settings  su    
 J	 
  #C+4>#(..))-DD+/+D+D+Q+Q+V+V+X#( # #"r   c                    / SQn/ SQnU R                  5       nU H  nXR;   ac  XS;   a  [        X   5      n[        XU5        M(  US:X  a-  US   X   :w  a"  U R                  X   UR	                  S0 5      S9  [        XX   5        Mk  [
        R                  " SR                  U5      5        M     [
        R                  " SU R                  5        35        g)	z
Update the router settings.
)r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r   rS  r   r1  r   r  r  )r  r  _allowed_settings_int_settings_existing_router_settingsr  _casted_values          r   update_settingsRouter.update_settings  s    


 %)$5$5$7!C''$'$4MD}5 1156HIV[X22-3[28** 73 3  Dv{3%++,G,N,Ns,ST' ( 	##&?@Q@Q@S?T$UVr   c                 X   US   S   n[        U5      nUS:X  a`  SR                  U5      nU R                  R                  USUS9nUc/  [        R
                  " XS9  U R                  R                  USUS9nU$ US:X  aX  UR                  S	5      SL a"  U S
3nU R                  R                  USUS9nU$ U S3nU R                  R                  USUS9nU$ UR                  S	5      SL a   U S3nU R                  R                  XeS9nU$ U S3nU R                  R                  XeS9nU$ )at  
Returns the appropriate client based on the given deployment, kwargs, and client_type.

Parameters:
    deployment (dict): The deployment dictionary containing the clients.
    kwargs (dict): The keyword arguments passed to the function.
    client_type (str): The type of client to return.

Returns:
    The appropriate client based on the given client_type and kwargs.
r  r  re  z{}_max_parallel_requests_clientTr  r  r  )r  r   r  r  _stream_async_client_async_client_stream_client)r  r  _client)r   r  r   r  r2    set_max_parallel_requests_clientr1  )r  r  r  rg  r  r  	cache_keyr  s           r   r  Router._get_client   s    l+D1+LV+T119@@JIZZ))$AQ * F ~%FF,0 --!dEU .  MG#zz(#t+'j(<=	--!dEU .  'j6	--!dEU .  zz(#t+'j7	--! .  'j0	--! .  r   c           	      	   [         R                  " SU 35        [        U5      n[        5       n [        R
                  " US9nSn	Sn
Sn[        U5      n[        5       nUR                  S5      nU SU 3nU R                  R                  US	US
9=(       d    0 n[        U5       GH  u  nnUR!                  S0 5      nUR!                  S0 5      n UR!                  SS5      nUc  UR!                  SS5      nU R#                  UUS9nU=(       d    UR!                  SS5      n[%        U[&        5      (       ab  UR!                  SS5      bO  [%        US   [(        5      (       a7  UUS   :  a.  UR+                  U5        S	n	U
SR                  UUS   U5      -  n
M  UR!                  SS5      nU R                  R                  US	US
9=(       d    Sn[%        U[&        5      (       a  U R.                  S:w  a  UR!                  US5      UU'   [1        UUU   5      n[%        U[&        5      (       aJ  UR!                  SS5      b7  [%        US   [(        5      (       a  US   U::  a  UR+                  U5        S	nGM  UbR  UR!                  S5      b@  UR!                  S5      nUb,  [3        [5        S!0 UD6US9(       d  UR+                  U5        GM(  Uc  GM.  [        R6                  SL d  GMD  [        R8                  " U[5        S!0 UD6S9u  nn  n[        R:                  " UUS9nUc  GM  [        R<                  R?                  US9nS/nURA                  5        H@  u  n n!U U;  d  M  U U;   d  M  [         R                  " SU  35        UR+                  U5        MB     GM     [C        U5      [C        U5      :X  a9   US	L a	  [E        US9eU	S	L a%  [        RF                  " SR                  U
5      USS 9e[C        U5      S:  a(  [        U5       V"V#s/ s H  u  n"n#U"U;  d  M  U#PM     nn"n#[C        U5      S:  a  [        R<                  RI                  U5      nU$ ! [         a:  n[         R                  " SR                  [        U5      5      5        Us SnA$ SnAff = f! [         a9  n[         R,                  " SR                  [        U5      5      5         SnAGNCSnAff = fs  sn#n"f )"a  
Filter out model in model group, if:

- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- [TODO] function call and model doesn't support function calling
z2Starting Pre-call checks for deployments in model=)r  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr  rQ  z:rpm:Tr  r   r  r;  rJ  r   rU  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r  r   r   rM  allowed_model_region)r   r  r<  r  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=rx  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r  r{   )%r   r  r  r  r   token_counterr  rT  r  r   r   rp   rY  r   r  r  r1  rX  r  r  r   r  r  r   maxrq   rT   drop_paramsrm   rf  rt  get_non_default_paramsr}  r  r]   r  _get_order_filtered_deployments)$r  r   r   r  r  _returned_deploymentsinvalid_model_indicesinput_tokensr  _context_window_error_potential_error_str_rate_limit_errorr  re  rS  rh  model_group_cacher  r  r  rv  r;  r  r  current_request_cache_localcurrent_requestr  r/  r  rZ  r  special_paramsr!  r"  ids$                                       r   _pre_call_checksRouter._pre_call_checks6  sa     	##@H	
 !%%8 9 #	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>?OC(nn-=rBO$..r:KW(__\4@
%!0!4!4\4!HJ!77)u 8 
 #Ho&9&9'4&H z400"'94@L #:.@#A3GG(:6H+II-11#604-,CJJ %z2D'E|,
 ! #tR0H 

$$ TDT %   	 ( ,d33))-EE.?.C.CHa.P!(+"%/1B81L#
 55'++E48D #?5#93??+E2oE-11#6,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .11#6  )g.A.AU.J3:3K3K0Q0Q40*Aq +2*M*M5H+' +2 *1)M)M&4 *N *& '8%8N 2 8 8 :1$;;^@S177"STUSV W 255c: !;K  @Z $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) &&;<%<DAq11 < " % $%)$+MM$Q$Q%%! %$M  	)!''~  F  FF
 )(	)l  W%//0F0M0McRSf0UVVWH%sB   Q CR S& S&
R#/RRR 
S#*.SS#c                     XR                   ;  a  gU R                   U   n[        U[        5      (       a  UnU$ US   nU$ )zt
Get the model from the alias.

Returns:
- str, the litellm model name
- None, if model is not in model group alias
Nr   )r   r  r   )r  r   _items      r   rz  Router._get_model_from_alias  sL     ...&&u-eS!!E  'NEr   c                 `    U R                    Vs/ s H  o"S   S   U:X  d  M  UPM     sn$ s  snf )z&
Get the deployment by litellm model.
r   r   )r   )r  r   r  s      r    _get_deployment_by_litellm_model'Router._get_deployment_by_litellm_model  s0      ??T?a0@.A'.Je.S?TTTs   ++r  c                    SnUb]  UR                  S5      =(       d    0 nUR                  S5      =(       d    0 nUR                  S5      =(       d    UR                  S5      nUSL a  XR                  US94$ U R                  U5      (       aG  U R                  US9n	U	b&  U	R                  R
                  n
XR                  SS94$ [        S	U S
35      eU R                  US9nUb  UnXR                  ;  a  U R                  R                  US9nU(       a  X4$ Ub5  X`R                  ;   a&  U R                  U   R                  US9nU(       a  X4$ U R                  bD  U R                  R                  5       nU R                  S   R                  5       US'   XS   S'   X4$ U R                  US9n[!        U5      S:X  a  U R                  US9n["        R$                  " SU 35        [!        U5      S:X  a  U R'                  5       (       aE  U R)                  5       nU(       a.  ["        R*                  " SU SU S35        UnU R                  US9n[!        U5      S:X  aQ  U R-                  US9c  SU S3R/                  U5      nOSU S3R/                  U5      n[0        R2                  " UUSS9e[0        R4                  (       a'  U[0        R4                  ;   a  [0        R4                  U   nX4$ )a  
Common checks for 'get_available_deployment' across sync + async call.

If 'healthy_deployments' returned is None, this means the user chose a specific deployment

Returns
- str, the litellm model name
- List, if multiple models chosen
- Dict, if specific model chosen
Nr   rp  user_api_key_team_idTrx  rI  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z does not exist in Model ID mapr   r   r  r   zinitial list of deployments: zModel 'z7' not found. Attempting to use default fallback model 'z'.zYou passed in model=z+. There is no 'model_name' with this stringz1. There are no healthy deployments for this modelr  r  )r1  r  r  rW  r   r   r  r  rz  r   r   get_deployments_by_patternr   r   r  r  r  r   r  r  r=  r  r
  r  r   re  model_alias_map)r  r   r  r?  r  r  request_team_idr   rp  r  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr   fallback_modelr  s                    r   r  *Router._common_checks_available_deployment  so   & *.%%))*5;H-112DEK&ll& >!%%&<=  $&??e?LLLu%%,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[[z{  !66U6C(%E((("&"5"5"P"P #Q # #11  +#'@'@@&*&?&?#',, -  $
 ' 55 &&2%)%<%<%A%A%C"7;7N7N$8$& ##34 AF#34W=00 #7757I"#q("&"G"Ge"G"T##+,?+@A	
 "#q(**,,!%!A!A!C!)..!%(_`n_ooqr +E*.*C*Cu*C*U' &'1,&&%&8@ 4UG;fgnnG !5UG;lmttG --#!#  ""u0G0G'G++E ))r   c           	        #    SSK Jn  U R                  UUUUUS9u  pU" UUS9n[        U[        5      (       a  U$ [        XS9I Sh  vN n	[        R                  " SU	 35        [        R                  " SU	 35        U R                  UU	S	9nU R                  UUUb  [        [        [           U5      OSUUS
9I Sh  vN nU R                  (       a*  Ub'  U R                  U[        [        [           U5      UUS9n[!        U UUUU R#                  U5      S9I Sh  vN n[%        U5      S:X  a  ['        U UUS9I Sh  vN n
U
eU$  GN N N/ N7f)z
Get the healthy deployments for a model.

Returns:
- List[Dict], if multiple models chosen
*OR*
- Dict, if specific model chosen
r   )filter_team_based_modelsr   r  r?  r  r  )r   r  r  Nzasync cooldown deployments: zcooldown_deployments: r   cooldown_deploymentsr  r   r   r  r  )llm_router_instancer   r  r   ru  r  r   r  )r  r  r  r  r  r7   r   r  _filter_cooldown_deploymentsr  r   r   rG   r   r  r
   r,   r  r  r?   )r  r   r  r  r?  r  r  r  r   r  r  s              r   r  $Router.async_get_healthy_deployments  s    " 	O%)%M%M 3) &N &
" 7 3)

 )400&&%D$(&
  
 	##*+?*@A	
 	##&<=Q<R$ST"?? 3!5 @ 

 %)$J$J 3:B:NT*+X6TX)- %K %
 
 &&8+?"&"7"7$(d5H$I!-	 #8 # %< $) 3#'#O#O$%
 
 "#q(A(,!1 I
 O""a 

$
sJ   AEE		A6E?E AEE!E E	EEEEc                   #    U R                   S:w  aS  U R                   S:w  aC  U R                   S:w  a3  U R                   S:w  a#  U R                   S:w  a  U R                  UUUUUS9$  [        U5      nU R                  UUUUUS9I Sh  vN nUb  UR                  nUR
                  nU R                  UUUUUUS	9I Sh  vN n[        U[        5      (       a  U$ [        R                  " 5       n	U R                   S:X  a2  U R                  b%  U R                  R                  UUUUS
9I Sh  vN n
OU R                   S:X  a2  U R                  b%  U R                  R                  UUUUS
9I Sh  vN n
OU R                   S:X  a3  U R                  b&  U R                  R                  UUUUUS9I Sh  vN n
O]U R                   S:X  a  [        U UUS9$ U R                   S:X  a0  U R                   b#  U R                   R                  UUS9I Sh  vN n
OSn
U
c  [#        U UUS9I Sh  vN nUe[$        R&                  " SU SU R)                  U
5       SU 35        [        R                  " 5       nX-
  n[*        R,                  " U R.                  R1                  [2        R4                  USUU	US95        U
$  GN- GN GN GNP GN N N! [6         a  n[8        R:                  " 5       nUbh  UR=                  SS5      nUbS  [>        R@                  " URB                  X4S9RE                  5         [*        R,                  " URG                  X5      5        UeSnAff = f7f)z
Async implementation of 'get_available_deployments'.

Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
r   r   r   r   r   r  r   r  r  r?  r  N)r   r  r  r?  r  r  rH  r   r  r?  )rH  r   r  r?  r  r  r   r   rH  r   r  $get_available_deployment for model: , Selected deployment:  for model: z2<routing_strategy>.async_get_available_deployments)r  r  rT  r  r  r  rd  r  )$r   r  r   async_pre_routing_hookr   r  r  r  r  r  r  r>  async_get_available_deploymentsrB  r@  r+   r   r?   r   r  r  r  r  r   r   r`   r!  r  r"  r#  r1  r  r  r  r  r  )r  r   r  r  r?  r  r  pre_routing_hook_responser   r  r  r  r  r%  r  r  r0  s                    r   rb  %Router.async_get_available_deployment  s     !!%==%%)99%%)==%%)@@%%500!$7- 1  	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t44****,J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L ((*H -I''BB(//&R%5)% C 	 O)#"&0  	"+"6"6"8),001FM*$$*::5 eg''#99!Q G!	s   A$M'!K J5	6K ?J8 K MAK ,J;-AK .J>/AK 1K2K M:K KK %K&BK 4M5K 8K ;K >K K K K 
MBMMMc                 ~   #    XR                   ;   a(  U R                   U   R                  UUUUUS9I Sh  vN $ g N7f)z
This hook is called before the routing decision is made.

Used for the litellm auto-router to modify the request before the routing decision is made.
r  N)r   r#  )r  r   r  r  r?  r  s         r   r#  Router.async_pre_routing_hook  sU       %%%**51HH-!$7 I    s   3=;=c                    U R                  UUUUS9u  p[        U[        5      (       a  U$ [        U5      n[	        XS9nU R                  UUS9nU R                  (       a  Ub  U R                  UUUUS9n[        U5      S:X  aG  U R                  US9n	U R                  R                  XS9n
[	        XS9n[        UU
U R                  US	9eU R                  S
:X  a'  U R                  b  U R                  R                  XS9nOU R                  S:X  a  [!        U UUS9$ U R                  S:X  a)  U R"                  b  U R"                  R                  UUUS9nOvU R                  S:X  a*  U R$                  b  U R$                  R                  UUUUS9nO<U R                  S:X  a*  U R&                  b  U R&                  R                  UUUUS9nOSnUca  [(        R*                  " SU S35        U R                  US9n	U R                  R                  XS9n
[	        XS9n[        UU
U R                  US	9e[(        R*                  " SU SU R-                  U5       SU 35        U$ )z2
Returns the deployment based on routing strategy
)r   r  r?  r  r  r  Nr  r   r  )	model_idsr  )r   r   r   cooldown_listr   r  r   r  r   )rH  r   r  r   r  r   r   z, No deployment availabler!  r"  )r  r  r  r   r9   r  r   r  r  r  r   get_min_cooldownr\   r   r   get_available_deploymentsr+   r@  r   r>  r   r  r  )r  r   r  r?  r  r  r   r  r  r*  _cooldown_time_cooldown_listr  s                r   r  Router.get_available_deployment  s&    &*%M%M 3	 &N &
" )400&&+L,
  9$( 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA# B N 7(,N ','+'B'B,	    L0T5J5J5V..HH! I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA# B N 7(,N ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 r   r  c                     [         R                  " SU 35        [        U5      nU Vs/ s H  nUS   S   U;  d  M  UPM     sn$ s  snf )a  
Filters out the deployments currently cooling down from the list of healthy deployments

Args:
    healthy_deployments: List of healthy deployments
    cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

Returns:
    List of healthy deployments
zcooldown deployments: r  r  )r   r  r  )r  r   r  cooldown_setr  s        r   r  #Router._filter_cooldown_deployments  s`     	##&<=Q<R$ST/0 2
1
,'-\A 1
 	
 
s
   AAc                      UR                  S0 5      R                  SS5      nUc  Ub  U R                  XB5        ggg! [         a,  n[        R                  " S[        U5       35         SnAgSnAff = f)z'
Tracks successful requests rpm usage.
r  r  Nz$Error in _track_deployment_metrics: )r1  r  r  r   rT  r   )r  r  r  r  r  r  s         r   rh   Router._track_deployment_metrics-  s~    		Y!~~lB7;;D$GH'&&  (    	Y!''*NsSTvh(WXX	Ys   9> 
A4"A//A4r  c                 B    [        UUU R                  U R                  S9$ )N)r  rH  r   r   )"_get_num_retries_from_retry_policyr   r   )r  r  rH  s      r   r>   (Router.get_num_retries_from_retry_policy>  s)     2#%)%B%B**	
 	
r   c                 V   U R                   nUc  g[        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R
                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ [        U[        R                  5      (       a  UR                  b  UR                  $ gg)z
BadRequestErrorRetries: Optional[int] = None
AuthenticationErrorRetries: Optional[int] = None
TimeoutErrorRetries: Optional[int] = None
RateLimitErrorRetries: Optional[int] = None
ContentPolicyViolationErrorRetries: Optional[int] = None
N)r   r  r   re  BadRequestErrorAllowedFailsr1  AuthenticationErrorAllowedFailsrl  TimeoutErrorAllowedFailsr,  RateLimitErrorAllowedFailsr  'ContentPolicyViolationErrorAllowedFails)r  r  r   s      r   get_allowed_fails_from_policy$Router.get_allowed_fails_from_policyH  s     >B=V=V' y'"9"9::$@@L'CCCy'"="=>>$DDP'GGGy'//22$==I'@@@y'"8"899$??K'BBBy'"E"EFF$LLX'OOO Y Gr   c                 F   SSK Jn  U R                  c  g U R                  nU" UR                  S/UR                  S9nX0l        [        R                  R                  U5        [        R                  R                  UR                  5        [        R                  " S5        g )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingrB  r   rD  webhook_urlslack_alerting_loggerr   r  r;  r  !response_taking_too_long_callbackr   r  )r  rB  router_alerting_config_slack_alerting_loggers       r   r  Router._initialize_alertingp  s    S'151E1E!.5HHY 6 B B"
 &<"((==>TU((EE"DD	
 	""H	
r   CustomRoutingStrategyc                 `    [        U SUR                  5        [        U SUR                  5        g)z
Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

Use this to set your custom routing strategy

Args:
    CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
r  rb  N)rS  r  rb  )r  rN  s     r   set_custom_routing_strategy"Router.set_custom_routing_strategy  s4     	&!::	

 	,!@@	
r   c                 N    S [         l        U R                  R                  5         g rE  )r   r   flush_cacherO  s    r   rS  Router.flush_cache  s    

 r   c                     / [         l        / [         l        / [         l        / [         l        S U l        U R                  5         g rE  )r   r  r  r  r
  r   rS  rO  s    r   resetRouter.reset  s6    #% *,'#% *,' r   )r`  rL  rW  ri  rb  r]  r   rG  r  ra  rJ  rH  r  r^  rZ  rY  rj  rV  r  rI  rM  r\  rK  rf  r   r  rc  r_  rg  r   r   rh  r  rX  r  rd  rN  ro  r   r   rs  r  r~  r  r  r  r  rr  r  r  r  r  r  r   r   r   r   r   r   r   r   r  r   r   r   r   r   r  r   r   r   r   r   r   r   r   r   r  r  r   r   r   r  rB  r@  r   r>  r   r   r   r   r   r   r   r   r  r   r   r   r   r   r[  r  r   r   r  r   r   r   r   rq  r   r   r   rI  r   r   r   r   r   ru  r  r  r  r  r  r  rt  r  r  r  r  r  )F)r   rE  )FFF)T)r  r  )NNNN)NNNNN)NF)NNN)NNFN)NNF)r|   r}   r~   r   r  r   __annotations__r   r   r  r   r   tenacityr   r&   r   r)   r   r   r   r   r	   r   rZ   rS   r
   r   rO   r_   r  r  floatr   r[   rX   rN   rW   rb   rM   r!  r  r-  staticmethodr   r   r   r4  r^   r   r  rk  r  r  r   r  r  rj   rh   r  r  r   rG   r  r_  r  r  r  rR   r  r  ri  r  r  r  r  r  r  r  r   r  r  r  r)  r'  r3  r2  rH   r:  r9  rB  rH  rF  rP  rS  rT  r]  r\  rg  rl  rs  rz  rr  r  ri   r  r  r  r  rI   r  r  rc   r  r  r  r  r  rF  r  r  r   r  r  rO  r"   wrapr  r  r  r  r  r  r  r  r=  r  r  r	  r  r  r  r  rt   r  r  r  r  r  r  r  rk  r  r  r  rT   r  r  r  r  r   r  r  r  r  r  r  r*  rW  r/  r3  rg  rX  r?  rG  rd   rO  rV   rq  rv  r  r   r   r  r  r(  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r  rz  r  r  r  rb  rv   r#  r  r  rh  r>   r?  r  rQ   rP  rS  rV  r   r{   r   r   r   r      s#   uK&+OXd^+&11H:>h67>:>h67>MQeL(C,G&H!IJQ ;?<@#'$($((,*/ ,0*.%) #'*. 7;!06 )+)+ ',%*     ,0 DH&(DH48 "#+0_m&$*+T$sCx.-AAB
m& $$78m& t$789m& C=m& SMm& SMm& !m& "$m& m&  !K
!m&& 'm&* #5/+m&, #3--m&0 c]1m&2  
3m&8 %9m&: !;m&< !)!
=m&B (0}Cm&D Em&F _-Gm&H $I
Im&N Om&P #'Qm&R #'Sm&T $eC!::;;<
Um&Z !%[m&\ #]m&^ _m&` +t#$
am&f #'#
gm&l  
mm&r '
sm&x  
ym&~ $D>m&@ "&
Am&P #++@"AQm&R  $Sm&T !))@ AUm&V ".1Wm&X "*!"
Ym&^ %)_m&` 
am&^ D 	.38n	.	z,,	-	. 	.+ +9 %os&: ;9TX9v	F;
zj
X1$ U(01F(GU,4 2$(c3h$8	}11	2&BB$(c3h$8B	}11	2BL $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3jB>+B> tCH~&B> 	B>
 
B>HQQ$(c3h$8Q	}12	3Qn 1;	

 
 !)	

 

, EOPP4<SMP	P, NR''(,'=Ec]'	'H (,	+
+
 +
  }	+

 
+
Z t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	M,M, tCH~&M, 	M,f di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  << '(< 	<|;; ; $	;
 CHo; S#X;z
X 
X 
XZBZB '(ZB S#X	ZBxs 3 , ,C ,\c # *Fs F3 FP( (3 (TD) DC DLT3 Ts T3 Tl3 *)C )^ $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JDS D# DT $)&+#(  4.	
 d^ 4.:DS D DL
x 


5=
 -5>38n-0>A	c3h>OO5=ObAA-5AN $)	 S$Y 4.	 
$6c4i 0 6 6x $(	 S$Y 4.	 
2DuS$Y'7 D DN 
	0BB 
	BH 
	>NN 
	Nd  $v}v 
	vp//l .21#1 &c]1x E_#_ = ?
_H .21#1 &c]1
#
2 .2(,	
#
 &c]
 %	
,}!}! $D>}! D>	}!
 #+4.}! #+4.}! c]}! }! }!~ [[]0 0j &*$(3737-- c]- D>	-
 #+4.- #+4.-^ [[]J% J%X $ :>!!)1#!L /3*.3737,0@@ &d^@ "$	@
 #+4.@ #+4.@ $D>@DW &*$S$s)^,-$ c]$ 
$s)		$2Xc] $ /3*.// / 	/
 &d^/ "$/ 
sEz	/f@D! 
#!FJ 
JX 
+3C= 
D>>	/	0> # # #t #J 4<TN	6 $$$1$;?$	$L5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '>KK K 	K
 K 
*	KZ D -@j -@^&z &d &P-A -Ads* s sj:$:;>:GJ:x 8L 6$F$F*-$F	$FN 6:JJ%-c]J	J4/J /8J;O /bC HZ,@ 4s x
/C &	(3 	(8D> 	( #	*	0 EI58>B	 
 58>A	  !	PTNP !P SM	P
 
Pd $  
:# 
:(4. 
:66),6	)	6pf f >Af 	.	!f P
 
8P 
>G$G$	x}hsm+	,G$R 1266	.	!6 36 cSVh B ;?""*23-"	"HN$ N4 N R$ R4 MR""3-"EI"	c"HE E Ec C HSM & FJ&*5=c]	& &*!%	33 c]3 #	3
 
!	"3jx} S	 0 IM-8@	#*
>(4. 
>T 
> +/"3-	!	"@ JN("3-(9A#(	$*+	,(X %),0!%	 SM  %SM  #	 
 
c49n	 D"%	.#B1Wf4v *.B%B% "B% tCH~&	B%
 !B%H3 8C= &Uc Ud U 48,0.3)-w*w* 4S#X/0w* c4i()	w*
 &d^w* !w* 
sE$*%%	&w*z 48,0.3+/U#U# U# 4S#X/0	U#
 c4i()U# &d^U# #4.U# 
tDz4	 U#v 48,0.3ZZ Z 4S#X/0	Z
 c4i()Z &d^Z@ 48,0.3  4S#X/0	
 c4i() &d^ 
(	)< 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{z
#':
EI#Y
	d
. FJY,4TNY$ BF
"
19#
&Py &PP
0
%>
.!r   r   )r  r  enumr  r&  r  r   r  r  r"  r  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   r   httpxr   r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm._uuidr   litellm.caching.cachingr   r   r   r   litellm.constantsr   "litellm.integrations.custom_loggerr   #litellm.litellm_core_utils.asyncifyr   r  r   r   ,litellm.litellm_core_utils.coroutine_checkerr    .litellm.litellm_core_utils.credential_accessorr!   %litellm.litellm_core_utils.dd_tracingr"   *litellm.litellm_core_utils.litellm_loggingr#   r  0litellm.litellm_core_utils.sensitive_data_maskerr$   &litellm.router_strategy.budget_limiterr%   "litellm.router_strategy.least_busyr&   #litellm.router_strategy.lowest_costr'   &litellm.router_strategy.lowest_latencyr(   &litellm.router_strategy.lowest_tpm_rpmr)   )litellm.router_strategy.lowest_tpm_rpm_v2r*   &litellm.router_strategy.simple_shuffler+   )litellm.router_strategy.tag_based_routingr,   /litellm.router_utils.add_retry_fallback_headersr-   r.    litellm.router_utils.batch_utilsr/   r0   r1   /litellm.router_utils.client_initalization_utilsr2   2litellm.router_utils.clientside_credential_handlerr3   r4   #litellm.router_utils.cooldown_cacher5   &litellm.router_utils.cooldown_handlersr6   r7   r8   r9   r:   ,litellm.router_utils.fallback_event_handlersr;   r<   r=   *litellm.router_utils.get_retry_from_policyr>   r7  !litellm.router_utils.handle_errorr?   r@   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkrA   Clitellm.router_utils.pre_call_checks.responses_api_deployment_checkrB   >litellm.router_utils.router_callbacks.track_deployment_metricsrC   rD   litellm.schedulerrE   rF   litellm.types.llms.openairG   rH   rI   rJ   litellm.types.routerrK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   litellm.types.servicesr`   litellm.types.utilsra   rb   rc   rd   rg  re   rf   rg   rK  rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   &router_utils.pattern_match_deploymentsrs   opentelemetry.tracert   _Spanr  ru   rv   Enumrx   r   r{   r   r   <module>r     s             #         &  ! 9 " 2   9 ; B K M 8 P P G F H N J P A M 
 R >  
 2      . 0 
 * 9 
   G1
 DDJ $)) gw gwr   