
    Mi,j             	       r   S SK r S SKrS SKJrJr  S SKJrJrJrJrJ	r	J
r
JrJrJrJr  S SKrS SKJrJr  S SKJr  S SKJrJr  S SK7  S SKJr  S S	KJr  S S
KJr  \(       a  S SKJr  O\r " S S\5      r  SS\
\!   S\"S\\!   S\#4S jjr$S\#S\	\!\!4   4S jr%S\
\!   S\!S\&4S jr'S\
\!   S\	\!\!4   4S jr(g)    N)datetime	timedelta)
TYPE_CHECKINGAny	AwaitableCallableDictListLiteralOptionalTuplecast)print_verboseverbose_loggerCustomLogger)LiteLLM_TeamTableUserAPIKeyAuth)*_sanitize_prometheus_label_nameStandardLoggingPayload)!get_end_user_id_for_cost_tracking)AsyncIOSchedulerc                   0   \ rS rSrS rS\\\\   4   4S jrS\S\	4S jr
S\S\\   4S jrS\S	\\   S\\   4S
 jrS\S\\\\   4   4S jrS\S	\\   4S jrS\	SS4S jrS\S\\   S\\   SS4S jrS\S\SS4S jrS\4S jrS\\\\   4   SS4S jrS\S\4S jrS rS\S\\   4S jrS rS\S\\   S\\   S \\   S!\\   S"\\   S#\\   S$\\   S%\4S& jr S"\\   S#\\   S\\   S \\   S'\!S(\"4S) jr#S\\   S\\   S \\   S!\\   S"\\   S#\\   S$\\   S(\"S%\4S* jr$S\\   S \\   S+\!S,\!4S- jr%S+\!S!\\   S\\   S \\   S"\\   S#\\   S%\4S. jr&S/ r' SwS0\!S1\(S2\)S3\\   4S4 jjr*S5\!S2\)4S6 jr+S7\!4S8 jr, SxS7\!S%\S9\"4S: jjr-\.S;\(S\4S< j5       r/S=\S+\!S1\(4S> jr0S=\S+\!S1\(4S? jr1S@\2SA\SB\\   SC\\   SD\4
SE jr3SA\SB\SC\SD\4SF jr4SA\SB\\   SC\\   SD\4SG jr5SA\SB\\   SC\\   SD\4SH jr6SA\SB\SC\SD\SI\4
SJ jr7SK\4SL jr8SM\SN\"SO\"4SP jr9SQ\\"   SN\\"   S\"4SR jr:SS\;ST\<\=\\>   \\2   4      4   SU\;\\>   /\<S   4   SV\?SW   4SX jr@SY rASZ rBS[ rCS\ rDS]\\E\\)4      4S^ jrFS_\\G   4S` jrHS"\\   S#\\   Sa\"Sb\"S(\"4
Sc jrISd\Se\SN\\"   SQ\\"   S(\"S\G4Sf jrJSg\G4Sh jrKS2\)4Si jrLS\\   S \\   S(\"Sj\"Sk\\"   4
Sl jrMS\S \Sj\"Sk\\"   S(\"S\)4Sm jrNSn\OS\"4So jrPSp\>Sq\>S\\"   4Sr jrQ\.Ss\R4St j5       rS\.Su 5       rTSvrUg)yPrometheusLogger#   c                 
    SSK JnJnJn  U R	                  5       U l        U R                  U5      U l        U R                  U5      U l        U R                  U5      U l	        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SS	U R                  S5      [        S
9U l        U R                  SSU R                  S5      [        S
9U l        U R                  SSU R                  S5      [        S
9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  SSU R                  S5      S9U l        U R                  S S!U R                  S 5      S9U l        U R                  S"S#U R                  S"5      S9U l        U R                  S$S%/ S&QS9U l        U R                  S'S(/ S&QS9U l        U R                  S)S*U R                  S+5      S9U l        U R                  S,S-U R                  S.5      S9U l        U R                  S/S0U R                  S/5      [        S
9U l        U R                  S1S2S3/S9U l         U R                  S4S5U R                  S45      S9U l!        U R                  S6S7U R                  S65      S9U l"        U R                  S8S9U R                  S85      S9U l#        U R                  S:S;U R                  S:5      S9U l$        U R                  S<S=U R                  S<5      S9U l%        U R                  S>S?U R                  S>5      S9U l&        U R                  S@SAU R                  S@5      5      U l'        U R                  SBSCU R                  SB5      5      U l(        U R                  SDSESF/S9U l)        U R                  SGSH/ SIQS9U l*        U R                  SJSKU R                  SJ5      S9U l+        g ! [X         a  n[[        SL[]        U5       35        UeS nAff = f)MNr   )CounterGauge	Histogram$litellm_proxy_failed_requests_metriczjTotal number of failed responses from proxy - the client did not get a success response from litellm proxy)namedocumentation
labelnames#litellm_proxy_total_requests_metriczXTotal number of requests made to the proxy server - track number of client side requests$litellm_request_total_latency_metricz0Total latency (seconds) for a request to LiteLLM)r&   bucketslitellm_llm_api_latency_metricz1Total latency (seconds) for a models LLM API call*litellm_llm_api_time_to_first_token_metricz-Time to first token for a models LLM API calllitellm_spend_metriczTotal spend on LLM requests)r&   litellm_total_tokens_metricz7Total number of input + output tokens from LLM requestslitellm_input_tokens_metricz.Total number of input tokens from LLM requestslitellm_output_tokens_metricz/Total number of output tokens from LLM requests$litellm_remaining_team_budget_metriczRemaining budget for teamlitellm_team_max_budget_metriczMaximum budget set for team*litellm_team_budget_remaining_hours_metricz*Remaining days for team budget to be reset'litellm_remaining_api_key_budget_metriczRemaining budget for api key!litellm_api_key_max_budget_metriczMaximum budget set for api key-litellm_api_key_budget_remaining_hours_metricz.Remaining hours for api key budget to be reset,litellm_remaining_api_key_requests_for_modelzLRemaining Requests API Key can make for model (model based rpm limit on key))hashed_api_keyapi_key_aliasmodel*litellm_remaining_api_key_tokens_for_modelzJRemaining Tokens API Key can make for model (model based tpm limit on key)litellm_remaining_requestszWLLM Deployment Analytics - remaining requests for model, returned from LLM API Provider!litellm_remaining_requests_metriclitellm_remaining_tokensz:remaining tokens for model, returned from LLM API Providerlitellm_remaining_tokens_metriclitellm_overhead_latency_metricz;Latency overhead (milliseconds) added by LiteLLM processing(litellm_provider_remaining_budget_metriczHRemaining budget for provider - used when you set provider budget limitsapi_providerlitellm_deployment_statezlLLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outagelitellm_deployment_cooled_downzLLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down$litellm_deployment_success_responseszOLLM Deployment Analytics - Total number of successful LLM API calls via litellm$litellm_deployment_failure_responseszLLM Deployment Analytics - Total number of failed LLM API calls for a specific LLM deploymeny. exception_status is the status of the exception from the llm api!litellm_deployment_total_requestszXLLM Deployment Analytics - Total number of LLM API calls via litellm - success + failure+litellm_deployment_latency_per_output_tokenz3LLM Deployment Analytics - Latency per output token'litellm_deployment_successful_fallbackszfLLM Deployment Analytics - Number of successful fallback requests from primary model -> fallback model#litellm_deployment_failed_fallbackszbLLM Deployment Analytics - Number of failed fallback requests from primary model -> fallback model(litellm_callback_logging_failures_metriczTTotal number of failures when emitting logs to callbacks (e.g. s3_v2, langfuse, etc)callback_name&litellm_llm_api_failed_requests_metricz5deprecated - use litellm_proxy_failed_requests_metric)end_userr7   r8   r9   team
team_aliasuserlitellm_requests_metriczdeprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, userz(Got exception on init prometheus client )/prometheus_clientr    r!   r"   _parse_prometheus_configlabel_filters_create_metric_factory_counter_factory_gauge_factory_histogram_factoryget_labels_for_metricr#   r'   LATENCY_BUCKETSr(   r*   r+   r,   litellm_tokens_metricr.   r/   r0   r1   r2   r3   r4   r5   r6   r:   r<   r>   r?   r@   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rL   rQ   	Exceptionr   str)selfkwargsr    r!   r"   es         y/var/www/eduai.edurigo.com/storigo/production/storigo_env/lib/python3.13/site-packages/litellm/integrations/prometheus.py__init__PrometheusLogger.__init__%   s   U	CC "&!>!>!@D %)$?$?$HD!"&"="=e"DD&*&A&A)&LD#8<8M8M; K55: 9N 9D5 8<7L7L:x559 8M 8D4 9=8O8O6B55: ( 9P 9D5 372I2I0C556VW'	 3J 3D/ ?C>U>U<?  55@ ( ?V ?D;" )-(=(=&-556LM )> )D% *.)>)>-I556ST *? *D& 04/D/D-@556ST 0E 0D, 150E0E.A556TU 1F 1D- 9=8K8K6+55: 9L 9D5 372E2E0-556VW 3F 3D/ ?C>Q>Q<<55@ ?R ?D; <@;N;N9.55= <O <D8 6:5H5H30557 6I 6D2 BFATAT?@55C BU BD> AE@S@S>^G AT AD= ?C>Q>Q<\G ?R ?D; 6:5H5H,i557 6I 6D2 483F3F*L555 4G 4D0 483J3J1M555 ( 4K 4D0 =A<O<O:Z*+ =P =D9 -1,?,?*~556PQ -@ -D) 372G2G0 ]556VW	 3H 3D/ 9=8M8M;o55: 9N 9D5 9=8M8M; @55: 9N 9D5 6:5J5J8x557 6K 6D2 @D?V?VBS55A @W @D< <@;P;P9x**+TU<D8 8<7L7L5t**+PQ8D4 =A<Q<Q?t+, =R =D9 ;?:O:O=U ;P ;D7 ,0+@+@. a556OP ,A ,D(  	DSVHMNG	s   SS 
T#S<<Treturnc                 R   SSK nSSKJn  UR                  nU(       d  0 $ [        R
                  " SU 35        / n[        5       U l        U HY  n[        U[        5      (       a	  U" S0 UD6nOUnUR                  U5        U R                  R                  UR                  5        M[     U R                  U5      nUR                  (       a:  U R                  U5        SSR!                  UR"                  5      -   n[%        U5      eU R'                  U5      n	U R)                  U	5        U	$ )zNParse prometheus metrics configuration for label filtering and enabled metricsr   N)PrometheusMetricsConfigzprometheus config: z!Configuration validation failed:

 )litellm%litellm.types.integrations.prometheusrf   prometheus_metrics_configr   debugsetenabled_metrics
isinstancedictappendupdatemetrics_validate_all_configurations
has_errors_pretty_print_validation_errorsjoinall_error_messages
ValueError_build_label_filters_pretty_print_prometheus_config)
r^   ri   rf   configparsed_configsgroup_configparsed_configvalidation_resultserror_messagerT   s
             ra   rS   )PrometheusLogger._parse_prometheus_config@  s   Q22 I26(;< "u"L,-- 7 G, G ,!!-0  ''(=(=> # ">>~N((001CD@499"55D M ]++ 11.A 	,,];    r}   c                 :   / n/ nU H  nUR                    Hv  nU R                  U5      nU(       a  UR                  U5        M.  UR                  (       d  MA  U R	                  XTR                  5      nU(       d  Me  UR                  U5        Mx     M     [        X#S9$ )z>Validate all metric configurations and return collected errors)metric_errorslabel_errors)rs   _validate_single_metric_namerq   include_labels_validate_single_metric_labelsValidationResults)r^   r}   r   r   r|   metric_namemetric_errorlabel_errors           ra   rt   -PrometheusLogger._validate_all_configurationsl  s    $F%~~#@@M!((6 ((("&"E"E#%:%:#K #{$++K8  . %  !}XXr   r   c                 j    SSK Jn  U[        U" [        5      5      ;  a  [	        UU" [        5      S9$ g)zValidate a single metric namer   )get_args)r   valid_metricsN)typingr   rm   DEFINED_PROMETHEUS_METRICSMetricValidationError)r^   r   r   s      ra   r   -PrometheusLogger._validate_single_metric_name  s8     	$c(+E"FGG('&'AB  r   labelsc                     SSK Jn  [        R                  U" [        U5      5      nU Vs/ s H  oUU;  d  M
  UPM     nnU(       a  [        UUUS9$ gs  snf )z#Validate labels for a single metricr   )r   r   invalid_labelsvalid_labelsN)r   r   PrometheusMetricLabels
get_labelsr   LabelValidationError)r^   r   r   r   r   labelr   s          ra   r   /PrometheusLogger._validate_single_metric_labels  sc     	  .88+[9

 .4QVEL7P%VQ''-) 
  Rs
   	AAc                     0 nU HK  nUR                    H8  nUR                  (       d  M  U R                  U5      b  M*  UR                  X$'   M:     MM     U$ )z1Build label filters from validated configurations)rs   r   r   )r^   r}   rT   r|   r   s        ra   rz   %PrometheusLogger._build_label_filters  sR    $F%~~(((88EM5;5J5J2	  . % r   c                     U R                  X5      nU(       aD  U R                  UR                  UR                  UR                  S9  [        UR                  5      eg)z
Ensure that all the configured labels are valid for the metric

Raises ValueError if the metric labels are invalid and pretty prints the error
r   T)r   "_pretty_print_invalid_labels_errorr   r   r   ry   message)r^   r   r   r   s       ra   "_validate_configured_metric_labels3PrometheusLogger._validate_configured_metric_labels  s[     99+N33'33*99(55 4 
 [0011r   r   Nc           	          SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SSS9nUR                  S	5        UR                  U" US
S95        UR                  (       a  UR                   Vs/ s H  oR                  PM     n	nUR                  S   R                  n
U" SSR                  U	5       3SS9nUR                  U" US
S95        U" SSSSSS9nUR                  SSSS9  [        U
5       H  nUR                  U5        M     UR                  U5        UR                   (       a  UR                    H  nU" SUR                   SSR                  UR"                  5       3SS9nUR                  U" US
S95        U" SUR                   S3SSSSS9nUR                  SSSS9  [        UR$                  5       H  nUR                  U5        M     UR                  U5        M     UR                  S	5        gs  snf ! [&         ai    UR                   H#  n[(        R*                  " UR,                  5        M%     UR                    H#  n[(        R*                  " UR,                  5        M%      gf = f)z-Pretty print all validation errors using richr   ConsolePanelTableTextu(   🚨🚨 Configuration Validation Errorsbold redstylerg   redborder_stylezInvalid Metric Names: ,    📊 Valid Metric NamesT
bold greenleftgreentitleshow_headerheader_styletitle_justifyr   Available Metricscyanr   no_wrapzInvalid Labels for '': u   🏷️ Valid Labels for ''Valid LabelsN)rich.consoler   
rich.panelr   
rich.tabler   	rich.textr   printr   r   r   rw   
add_columnsortedadd_rowr   r   r   ImportErrorr   errorr   )r^   r   r   r   r   r   consoler   r`   invalid_metricsr   metrics_error_textmetrics_tablemetricr   labels_error_textlabels_tabler   r   r   s                       ra   rv   0PrometheusLogger._pretty_print_validation_errors  s{   L	:,((&iG C:VE MM$MM%E:; "//+=+K+K#+KaMM+K   # !3 @ @!-  &*,TYY-G,HI$&" e$6UKL %3 $!-"(!(! (('vt )  %]3F!))&1 4 m, "../<<E(,.u/@/@.ATYYuOcOcEdDef()% MM%(9"NO#( :5;L;L:MQO$(%1&,%,$L !++N&RV+W!'(:(:!;$,,U3 "< MM,/' =* MM$i#l  	: 2 @ @$$\%9%9: !A1>>$$[%8%89  ?		:s&   A,H .G?E;H ?H A0I76I7r   r   c                     SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SU SSR                  U5       S	3S
S9n	U" SSSSSS9n
U
R                  SSSS9  [        U5       H  nU
R                  U5        M     UR                  S5        UR                  U" U	SS95        UR                  U
5        UR                  S5        g! [         a+    [        R                  " SU SU S[        U5       35         gf = f)z8Pretty print error message for invalid labels using richr   r   r   r   r   u%   🚨🚨 Invalid Labels for Metric: 'z'
Invalid labels: r   z'
Please specify only valid labels belowr   r   u$   🏷️ Valid Labels for this MetricTr   r   r   r   r   r   r   rg   r   r   zInvalid labels for metric 'r   z. Valid labels: N)r   r   r   r   r   r   r   r   rw   r   r   r   r   r   r   r   )r^   r   r   r   r   r   r   r   r   r   r   r   s               ra   r   3PrometheusLogger._pretty_print_invalid_labels_error  s   %	,((&iG 7}DWX\XaXabpXqWr  s[  \ E !< )$$L ##N&$#O-$$U+ . MM$MM%E:;MM,'MM$ 	  -k]#n=MM]^deq^r]st	s   CC 2C;:C;invalid_metric_namer   c           	          SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SU S3SS	9nU" S
SSSSS9n	U	R                  SSSS9  [        U5       H  n
U	R                  U
5        M     UR                  S5        UR                  U" USS95        UR                  U	5        UR                  S5        g! [         a(    [        R                  " SU S[        U5       35         gf = f)z=Pretty print error message for invalid metric name using richr   r   r   r   r   u   🚨🚨 Invalid Metric Name: 'z1'
Please specify one of the allowed metrics belowr   r   r   Tr   r   r   r   r   r   r   rg   r   r   zInvalid metric name: z. Valid metrics: N)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r^   r   r   r   r   r   r   r   r   r   r   s              ra   "_pretty_print_invalid_metric_error3PrometheusLogger._pretty_print_invalid_metric_errorE  s    %	,((&iG 12E1FFxy E "/ )$$M $$%8PT$U /%%f- 0 MM$MM%E:;MM-(MM$ 	  '(;'<<MfUbNcMde	s   B1B4 4/C&%C&c                     U R                  U5      nU(       a9  U R                  UR                  UR                  S9  [	        UR
                  5      eg)zM
Raises ValueError if the metric name is invalid and pretty prints the error
)r   r   N)r   r   r   r   ry   r   )r^   r   r   s      ra   _valid_metric_name#PrometheusLogger._valid_metric_namet  sQ     11+>33$)$5$5UEXEX 4  U]]++	 r   rT   c                     SSK Jn  SSKJn  SSKJn  SSKJn  U" 5       nU" SSS9nU" S	S
SSS9nUR                  SSS
S9  [        U S5      (       a?  U R                  (       a.  [        U R                  5       H  n	UR                  U	5        M     OUR                  S5        U" SS
SSS9n
U
R                  SSS
S9  U
R                  SSS9  U(       aN  [        UR                  5       5       H0  u  pU(       a  SR                  U5      OSnU
R                  X5        M2     OU
R                  SS5        UR                  S5        UR                  U" USS95        UR                  U5        UR                  U
5        UR                  S5        g!! [          a[    ["        R$                  " S[        U S5      (       a  [        U R                  5      OS 35        ["        R$                  " S U 35         g!f = f)"z>Pretty print the processed prometheus configuration using richr   r   r   r   r   z"Prometheus Configuration Processedz	bold bluer   u   📊 Enabled MetricsTzbold magentar   )r   r   r   r   zMetric Namer   r   rn   z8[yellow]All metrics enabled (no filter applied)[/yellow]u   🏷️  Label Filtersr   zAllowed Labelsyellowr   z[dim]No labels specified[/dim]z+[yellow]No label filtering applied[/yellow]z*[dim]All default labels will be used[/dim]rg   bluer   zEnabled metrics: zAll metricszLabel filters: N)r   r   r   r   r   r   r   r   r   hasattrrn   r   r   itemsrw   r   r   r   info)r^   rT   r   r   r   r   r   r   r   r   r   r   r   
labels_strs                 ra   r{   0PrometheusLogger._pretty_print_prometheus_config  s   @	C,((&iG =[QE ", +$	M $$]&$$Ot.//D4H4H$T%9%9:F!))&1 ; %%N
 !. )$	L ##M#N##$4H#E+1-2E2E2G+H'K " 		&)= 
 !((A ,I $$A@ MM$MM%F;<MM-(MM,'MM$ 	C#GDRcDdDdF4+?+?$@jw#xy /- AB	Cs   FF A"G43G4c                 f    [        U S5      (       d  gU R                  (       d  gXR                  ;   $ )z3Check if a metric is enabled based on configurationrn   T)r   rn   )r^   r   s     ra   _is_metric_enabled#PrometheusLogger._is_metric_enabled  s2     t.// ##2222r   c                    ^ ^ UU 4S jnU$ )zMCreate a factory function that returns either a real metric or a no-op metricc                     > U (       a  U S   OUR                  SS5      nTR                  U5      (       a  T" U 0 UD6$ [        5       $ )Nr   r$    )getr   
NoOpMetric)argsr_   r   metric_classr^   s      ra   factory8PrometheusLogger._create_metric_factory.<locals>.factory  sD    %)$q'vzz&"/EK&&{33#T4V44!|#r   rh   )r^   r   r   s   `` ra   rU   'PrometheusLogger._create_metric_factory  s    	$ r   c                     [         R                  U5      nXR                  ;  a  U$ U R                  U   nU Vs/ s H  oDU;   d  M
  UPM     nnU$ s  snf )z5
Get the labels for a metric, filtered if configured
)r   r   rT   )r^   r   default_labelsconfigured_labelsr   filtered_labelss         ra   rY   &PrometheusLogger.get_labels_for_metric  sl     0::;G 000!! !..{;  .
-e:K1KE~ 	 
 	
s   	AAc                   #    SSK Jn  [        R                  " SU 35        UR	                  S5      nUb  [        U[        5      (       d  [        SU 35      eUR	                  SS5      nUR	                  S0 5      =(       d    0 nUR	                  S	0 5      n	[        US
S9n
US	   S   nUS	   S   nUS	   S   nUS	   S   nUS	   S   nUS   nUS   nUS   nUS	   R	                  S5      nUS	   R	                  S5      n0 U(       a  UO0 EU(       a  UO0 EnUb  [        U[        5      (       a  US   nO/ n[        SU SU SU SU
 SU 3
5        [        S@0 SU
_SU_SU_SUS    _S US    _S!U_S"U_S#U_S$US	   S%   _S&S'_SU_S(U_S)U_S*US*   _S+US+   _S,US-   _S.S _S/S _S0[        US19_S2US	   R	                  S35      _6nUb9  [        U[        5      (       a$  UR                  S45      (       a  SS5KJn  U" U5      nU R!                  U
UUUUUUUUS69	  U R#                  UU
UUUUUUUS79	  U R%                  UUUUUUS89I S h  vN   U R'                  UUUU	S99  U R)                  UUUUUUUS:9  U R+                  XUUU5        US;   S<L aB  [-        U R/                  S=S>9US?9nU R0                  R2                  " S@0 UD6R5                  5         g g  N7f)ANr   r   z@prometheus Logging - Enters success logging function for kwargs standard_logging_objectz)standard_logging_object is required, got=r9   r   litellm_paramsmetadata
prometheusservice_typeuser_api_key_user_iduser_api_key_hashuser_api_key_aliasuser_api_key_team_iduser_api_key_team_aliascompletion_tokenstotal_tokensresponse_costrequester_metadatauser_api_key_auth_metadatarequest_tagsz'inside track_prometheus_metrics, model z, response_cost z, tokens_used z, end_user_id z, user_api_key rM   r7   r8   requested_modelmodel_grouprN   rO   rP   
user_emailuser_api_key_user_emailstatus_code200litellm_model_nametagsmodel_idapi_baserA   custom_llm_providerexception_statusexception_classcustom_metadata_labelsr   routeuser_api_key_request_routezsk-)
hash_token)	end_user_iduser_api_keyr  r9   user_api_teamuser_api_team_aliasuser_idr
  enum_values)	standard_logging_payloadr   r!  r  r9   r"  r#  r$  r%  )r"  r#  r!  r  r   r
  )r!  r  r_   r   )r_   r9   r!  r  r"  r#  r%  streamTr'   r   supported_enum_labelsr%  rh   )litellm.types.utilsr   r   rl   r   ro   rp   ry   r   r   UserAPIKeyLabelValuesget_custom_labels_from_metadatar]   
startswithlitellm.proxy.utilsr  ._increment_top_level_request_and_spend_metrics_increment_token_metrics#_increment_remaining_budget_metrics#_set_virtual_key_rate_limit_metrics_set_latency_metrics"set_llm_deployment_success_metricsprometheus_label_factoryrY   r'   r   inc)r^   r_   response_obj
start_timeend_timer   r&  r9   r   	_metadatar   r$  r!  r  r"  r#  output_tokenstokens_usedr
  _requester_metadatar  combined_metadata_tagsr%  r  _labelss                             ra   async_log_success_event(PrometheusLogger.async_log_success_event  s    >NvhW	

 FLZZ%F
  $+:$d4
 4
 ;<T;UV  

7B'$4b9?R"&&z26	7
 +:67MN/
;<OP5jABVW0<=ST6zB%
 11DE.~>0A.Fz.R.V.V /
 6N6

#*
+ 	#-
&9"r-
-G)R-
 $/J$d5
 5
 -^<EE5eW<L]O[ijuiv  wE  FQ  ER  Ra  bn  ao  p	
 , 
 
'
 -
 5]C	

 1?
 
 +
 
 0
;<UV
 
 
  %
 
 .j9
 .j9
  22GH!
" "#
$ !%
& $C*$'
, +:6::,-
8 $<--''..6%l3L 	;;#%1' 3'# 	< 
	
 	%% &>#%1' 3# 	& 	
  66' 3%1)' 7 
 	
 	
 	00%1	 	1 	
 	!!%1' 3 $ 	" 	
 	//+}	

 %X.$6.&*&@&@ E 'A ' (	G 44;;FgFJJL 7K	
s   IKKBKr&  r   r!  r  r9   r"  r#  r$  r%  c
                 6   [         R                  " S5        Ub  [        U[        5      (       a  US   n
[	        U R                  SS9U	S9n[	        U R                  SS9U	S9nU R                  R                  " S0 UD6R                  US   5        [	        U R                  SS9U	S9nU R                  R                  " S0 UD6R                  US	   5        [	        U R                  S
S9U	S9nU R                  R                  " S0 UD6R                  US   5        g )Nz2prometheus Logging - Enters token metrics functionr  r'   r(  r)  r-   r	  r.   prompt_tokensr/   r  rh   )r   rl   ro   rp   r6  rY   r[   r   r7  r.   r/   )r^   r&  r   r!  r  r9   r"  r#  r$  r%  r@  rA  s               ra   r1  )PrometheusLogger._increment_token_metrics  sW    	QR $/J$d5
 5
 -^<E*"&"<"<A #= # $	
 +"&"<"<9 #= # $	
 	""))4G488$^4	
 +"&"<"<9 #= # $	
 	((//:':>>$_5	
 +"&"<"<: #= # $	
 	))00;7;??$%89	
r   r   r
  c                   #    UR                  S0 5      R                  SS 5      nUR                  S0 5      R                  SS 5      nUR                  S0 5      R                  SS 5      n	UR                  S0 5      R                  SS 5      n
U R                  UUUU
U	S9I S h  vN   U R                  UUUUUS9I S h  vN   g  N  N7f)Nr   user_api_key_team_spenduser_api_key_team_max_budgetuser_api_key_spenduser_api_key_max_budget)r!  r  r
  key_max_budget	key_spend)r"  r#  
team_spendteam_max_budgetr
  )r   -_set_api_key_budget_metrics_after_api_request*_set_team_budget_metrics_after_api_request)r^   r"  r#  r!  r  r   r
  _team_spend_team_max_budget_api_key_spend_api_key_max_budgets              ra   r2  4PrometheusLogger._increment_remaining_budget_metrics  s      %((R8<<%t
 *--j"=AA*D
 (++J;?? $
 -00R@DD%t
 @@%1'.$ A 
 	
 	
 ==' 3",' > 
 	
 	
	
	
s$   BC!C"C<C=CCc
                 
   [        U R                  SS9U	S9n
U R                  R                  " S0 U
D6R	                  5         [        U R                  SS9U	S9n
U R
                  R                  " S0 U
D6R	                  U5        g )NrQ   r(  r)  r,   rh   )r6  rY   rQ   r   r7  r,   )r^   r   r!  r  r9   r"  r#  r$  r
  r%  rA  s              ra   r0  ?PrometheusLogger._increment_top_level_request_and_spend_metrics   s     +"&"<"<5 #= # $	
 	$$++6g6::<*"&"<"<2 #= # $	
 	!!((37377Fr   r_   r   c                    SSK Jn  U" U5      nSU 3nSU 3nUR                  U[        R                  5      =(       d    [        R                  n	UR                  U[        R                  5      =(       d    [        R                  n
U R
                  R                  XU5      R                  U	5        U R                  R                  XU5      R                  U
5        g )Nr   )#get_model_group_from_litellm_kwargszlitellm-key-remaining-requests-zlitellm-key-remaining-tokens-)	)litellm.proxy.common_utils.callback_utilsrZ  r   sysmaxsizer6   r   rm   r:   )r^   r!  r  r_   r   rZ  r   remaining_requests_variable_nameremaining_tokens_variable_nameremaining_requestsremaining_tokenss              ra   r3  4PrometheusLogger._set_virtual_key_rate_limit_metrics  s    	
 :&A-k]; 	) ,I)V& LL93;;GV3;; 	 LL7ET 	 	99@@k	

# 
!77>>k	

#
r   c                    UR                  S5      =(       d    [        R                  " 5       nUR                  S5      n	UR                  SS 5      n
UR                  SS 5      nU R                  U
US9nUbD  UR                  SS5      SL a/  U R                  R                  UUUUU5      R                  U5        O[        R                  " S	5        U R                  U
US9nUbB  [        U R                  S
S9US9nU R                  R
                  " S0 UD6R                  U5        U R                  U	US9nUbC  [        U R                  SS9US9nU R                  R
                  " S0 UD6R                  U5        g g )Nr:  r9  api_call_start_timecompletion_start_time)r9  r:  r'  FTzUTime to first token metric not emitted, stream option in model_parameters is not Truer*   r(  r)  r(   rh   )r   r   now_safe_duration_secondsr+   r   observer   rl   r6  rY   r*   r(   )r^   r_   r9   r!  r  r"  r#  r%  r:  r9  rd  re  time_to_first_token_secondsapi_call_total_time_secondsrA  total_time_secondss                   ra   r4  %PrometheusLogger._set_latency_metrics@  s    $ZZ
3Ex||~)/L)A
$jj)>E &

+BD I&*&A&A** 'B '
#
 (3

8U+t3;;BB"# g12  g '+&A&A* 'B '
# '2.&*&@&@ @ 'A ' (	G //66AAII+
 "88! 9 
 ).&*&@&@ F 'A ' (	G 55<<GwGOO" *r   c           	      \  #    SSK Jn  [        R                  " SU 35        UR	                  SS5      nUR	                  S0 5      nUR	                  S0 5      =(       d    0 n[        USS	9n	US
   S   n
US
   S   nUS
   S   nUS
   S   nUS
   S   nUR	                  SS 5         U R                  R                  U	UUUUUU
5      R                  5         U R                  U5        g ! [         a8  n[        R                  " SR                  [        U5      5      5         S nAg S nAff = f7f)Nr   r   z@prometheus Logging - Enters failure logging function for kwargs r9   r   r   r   r   r  r   r  r  r  r  r  	exception0prometheus Layer Error(): Exception occured - {})r+  r   r   rl   r   r   rL   r   r7  "set_llm_deployment_failure_metricsr\   rn  formatr]   )r^   r_   r8  r9  r:  r   r9   r&  r   r   r$  r!  r  r"  r#  r`   s                   ra   async_log_failure_event(PrometheusLogger.async_log_failure_event  sV    >NvhW	

 

7B';A::%r<
   $4b9?R7
 +:67MN/
;<OP5jABVW0<=ST6zB%
 	

;%	77>>"# ce33F; 	  	$$BII#a&Q 	s1   B#D,&A C' &D,'
D)1.D$D,$D))D,request_dataoriginal_exceptionuser_api_key_dicttraceback_strc                 @  #    SSK Jn   UR                  UUR                  S0 5      S9n[	        UR
                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  SS5      [        [        USS5      5      [        [        USS5      5      U R                  U5      UUR                  S	9n[!        U R#                  S
S9US9nU R$                  R&                  " S0 UD6R)                  5         [!        U R#                  SS9US9nU R*                  R&                  " S0 UD6R)                  5         g! [,         a8  n	[.        R0                  " SR3                  [        U	5      5      5         Sn	A	gSn	A	ff = f7f)a  
Track client side failures

Proxy level tracking - failed client side requests

labelnames=[
            "end_user",
            "hashed_api_key",
            "api_key_alias",
            REQUESTED_MODEL,
            "team",
            "team_alias",
        ] + EXCEPTION_LABELS,
r   StandardLoggingPayloadSetupproxy_server_requestr   r{  r9   r   r  N)rM   rP   r  r7   r8   rN   rO   r  r  r  r  r  r  r#   r(  r)  r'   ro  rh   )*litellm.litellm_core_utils.litellm_loggingrz  _get_request_tagsr   r,  r   r$  r  api_key	key_aliasteam_idrO   r]   getattr_get_exception_class_namerequest_router6  rY   r#   r   r7  r'   r\   r   rn  rq  )
r^   rt  ru  rv  rw  rz  r@  r%  rA  r`   s
             ra   async_post_call_failure_hook-PrometheusLogger.async_post_call_failure_hook  s    *	
(	/AA+%1%5%56Lb%Q B E 0*66&..,77088/99&..,77 , 0 0" =(:M4 PQ!$W-?PT%U!V $ > >?Q R'55K /&*&@&@ F 'A ' (	G 55<<GwGKKM.&*&@&@ E 'A ' (	G 44;;FgFJJL 	$$BII#a&Q 		s/   FEE F
F#.FFFFdatac                 H  #     SSK Jn  [        UR                  UR                  UR
                  UR                  SS5      UR                  UR                  UR                  UR                  SUR                  UR                  UUR                  S0 5      S9S9n[        U R                  S	S
9US9nU R                  R                   " S0 UD6R#                  5         g! [$         a8  n[&        R(                  " SR+                  [-        U5      5      5         SnAgSnAff = f7f)z`
Proxy level tracking - triggered when the proxy responds with a success response to the client
r   ry  r9   r   r  r{  r|  )rM   r7   r8   r  rN   rO   rP   r  r  r  r  r'   r(  r)  ro  Nrh   )r}  rz  r,  r   r  r  r   r  rO   r$  r  r  r~  r6  rY   r'   r   r7  r\   r   rn  rq  r]   )r^   r  rv  responserz  r%  rA  r`   s           ra   async_post_call_success_hook-PrometheusLogger.async_post_call_success_hook  s    !	 0*66088/99 $" 5&..,77&..,77!'550BB#')-2H")M C K  /&*&@&@ E 'A ' (	G 44;;FgFJJL 	$$BII#a&Q 		s/   D"CC D"
D'.DD"DD"request_kwargsc                 $    [         R                  " S5        UR                  S0 5      nUR                  S0 5      =(       d    0 nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  SS5      nUR                  S	S5      nUR                  S
S5      n	[        UUUU	U(       a  [	        [        USS5      5      OSU(       a  U R                  U5      OSUUS   S   US   S   US   S   US   S   UR                  S/ 5      S9n
 U R                  U=(       d    SUUU	=(       d    SS9  UbA  [        U R                  SS9U
S9nU R                  R                  " S0 UD6R                  5         [        U R                  SS9U
S9nU R                  R                  " S0 UD6R                  5         g! [         a8  n[         R                  " SR                  [	        U5      5      5         SnAgSnAff = f)z
Sets Failure metrics when an LLM API call fails

- mark the deployment as partial outage
- increment deployment failure responses metric
- increment deployment total requests metric

Args:
    request_kwargs: dict

(setting remaining tokens requests metricr   r   r9   Nr  r  r  rn  r  r  r   r  r  r  r  r  )r  r  r  rA   r  r  r  r7   r8   rN   rO   r  r   r  r  r  rA   rE   r(  r)  rF   zLPrometheus Error: set_llm_deployment_failure_metrics. Exception occured - {}rh   )r   rl   r   r,  r]   r  r  set_deployment_partial_outager6  rY   rE   r   r7  rF   r\   rq  )r^   r  r&  _litellm_paramsr  r  r  r  rn  llm_providerr%  rA  r`   s                ra   rp  3PrometheusLogger.set_llm_deployment_failure_metrics  sU   J	  !KL?M?Q?Q)2@$ -001A2FL"O!/!3!3GT!B266}dKK/33JEH/33JEH&**;=I*../DdKL 0#5!!)DMC	=$?@SW BKD229=PT +7
C'  7zB( .j9:PQ3J?- .11."E-K2 ..#5#;!!)/R	 /  $2*.*D*D$J +E + !,	 99@@K7KOOQ.&*&@&@ C 'A ' (	G 2299DGDHHJ 	  ^eeF 	s   G
G 
H.H

Hr<  c                     [         R                  " S5        UR                  S5      nUc  g US   nUR                  S0 5      =(       d    0 nUR                  S0 5      n	UR                  SS 5      n
UR                  SS 5      nU	R                  S5      =(       d    0 nUR                  S	S 5      nS nS nUS
   S   =n(       a$  UR                  SS 5      nUR                  SS 5      nUS
   R                  S5      =n(       aE  [        U R	                  SS9US9nU R
                  R                  " S0 UD6R                  US-  5        U(       aC   [        U R	                  SS9US9nU R                  R                  " S0 UD6R                  U5        U(       aB  [        U R	                  SS9US9nU R                  R                  " S0 UD6R                  U5         U R                  U
=(       d    SU=(       d    SU=(       d    SU=(       d    SS9  [        U R	                  SS9US9nU R                  R                  " S0 UD6R                  5         [        U R	                  SS9US9nU R                  R                  " S0 UD6R                  5         X2-
  nS nUR                  SS 5      b  US   SL a  UR                  SU5      U-
  nU=(       d    UnUR                  5       nS nUbO  US:  aH  UU-  n[        U R	                  SS9US9nU R                   R                  " S0 UD6R                  U5        g g g ! ["         a8  n[         R$                  " SR'                  [)        U5      5      5         S nAg S nAff = f) Nr  r   r  r   r   r9   r  
model_infoidhidden_paramsadditional_headersx_ratelimit_remaining_requestsx_ratelimit_remaining_tokenslitellm_overhead_time_msr?   r(  r)  i  r<   r>   r   r  rD   rF   r'  Tre  r   rG   zLPrometheus Error: set_llm_deployment_success_metrics. Exception occured - {}rh   )r   rl   r   r6  rY   r?   r   rh  r<   rm   r>   set_deployment_healthyrD   r7  rF   total_secondsrG   r\   rn  rq  r]   )r^   r  r9  r:  r%  r<  r&  r  r  r;  r  r  _model_infor  r`  ra  r  r  rA  response_ms!time_to_first_token_response_time_latency_latency_secondslatency_per_tokenr`   s                            ra   r5  3PrometheusLogger.set_llm_deployment_success_metricsq  s   I	  !KL""#<= % (//
;H,001A2FL"O'++J;I!/!3!3GT!B*../DdKL#--5;K"tT2H04.2%=o%N$& !  &8%;%;4d&" $6#9#92D$  ,D,c,-.' . 3*.*D*D$E +E + !,	 44;;FgFNN,t3 " 3*.*D*D$G +E + !,	 66==HHLL&  2*.*D*D$E +E + !,	 44;;FgFJJ$ ''#5#;!R!R)/R	 (  /&*&@&@ F 'A ' (	G 55<<GwGKKM.&*&@&@ C 'A ' (	G 2299DGDHHJ &.%:KEI- ""8T2>"8,4 #&&'>IJV 2 #D"R{H'557 !%(]Q->$4}$D!2*.*D*D$Q +E + !,	 @@GG '+, .?(  	$$^eeF
 	s   *L8 LL8 8
M:.M55M:rn  c                     Sn[        U S5      (       a  [        U S5      =(       d    Sn[        U5      S:  a  US   R                  5       USS  -   S-   nXR                  R
                  -  nU$ )Nr   r     r   .)r   r  lenupper	__class____name__)rn  exception_class_names     ra   r  *PrometheusLogger._get_exception_class_name  s|    !9n--#*9n#E#K  #$)$Q'--/2Fqr2JJSP ! 	 3 3 < <<##r   original_model_groupc                   #    SSK JnJn  [        R                  " SUU5        UR                  S0 5      nUR                  US9nUR                  S5      n[        [        [           UR                  S5      =(       d    / 5      n	[        UUUS   US	   US
   US   [        [        USS5      5      U R                  U5      U	S9	n
[        U R                  SS9U
S9nU R                  R                   " S0 UD6R#                  5         g7f)z6

Logs a successful LLM fallback event on prometheus

r   StandardLoggingMetadatarz  zLPrometheus: log_success_fallback_event, original_model_group: %s, kwargs: %sr   r  r9   r  r  r  r  r  r  N	r  fallback_modelr7   r8   rN   rO   r  r  r  rH   r(  r)  rh   )r}  r  rz  r   rl   r   get_standard_logging_metadatar   r
   r]   r,  r  r  r6  rY   rH   r   r7  )r^   r  r_   ru  r  rz  r;  standard_metadata
_new_modelr@  r%  rA  s               ra   log_success_fallback_event+PrometheusLogger.log_success_fallback_event  s     	

 	Z 	

 JJz2.	'EE" F  	
 ZZ(
T#Y

6 2 8b9+0%,-@A+,@A"#9:()BC );]D!QR ::;MN

 +"&"<"<E #= # $	
 	44;;FgFJJL   DDc                   #    SSK JnJn  [        R                  " SUU5        UR                  S5      nUR                  S0 5      n[        [        [           UR                  S5      =(       d    / 5      nUR                  US9n	[        UUU	S   U	S	   U	S
   U	S   [        [        USS5      5      U R                  U5      US9	n
[        U R                  SS9U
S9nU R                  R                   " S0 UD6R#                  5         g7f)z0
Logs a failed LLM fallback event on prometheus
r   r  zLPrometheus: log_failure_fallback_event, original_model_group: %s, kwargs: %sr9   r   r  r  r  r  r  r  r  Nr  rI   r(  r)  rh   )r}  r  rz  r   rl   r   r   r
   r]   r  r,  r  r  r6  rY   rI   r   r7  )r^   r  r_   ru  r  rz  r  r;  r@  r  r%  rA  s               ra   log_failure_fallback_event+PrometheusLogger.log_failure_fallback_eventC  s     	

 	Z 	

 ZZ(
JJz2.	T#Y

6 2 8b9'EE" F  	 ,0%,-@A+,@A"#9:()BC );]D!QR ::;MN

 +"&"<"<A #= # $	
 	0077B'BFFHr  stater  r  r  rA   c           
          [        U R                  SS9[        UUUUS9S9nU R                  R                  " S0 UD6R                  U5        g)z
Set the deployment state.
rB   r(  r  r)  Nrh   )r6  rY   r,  rB   r   rm   )r^   r  r  r  r  rA   rA  s          ra   set_litellm_deployment_state-PrometheusLogger.set_litellm_deployment_statep  s^     +"&"<"<6 #= # .#5!!)		

 	%%,,7w7;;EBr   c                 *    U R                  SXX45        g )Nr   r  r^   r  r  r  rA   s        ra   r  'PrometheusLogger.set_deployment_healthy       	))!X	
r   c                 *    U R                  SXX45        g )Nr  r  r  s        ra   r  .PrometheusLogger.set_deployment_partial_outage  r  r   c                 *    U R                  SXX45        g )N   r  r  s        ra   set_deployment_complete_outage/PrometheusLogger.set_deployment_complete_outage  r  r   r  c                 Z    U R                   R                  XX4U5      R                  5         g)z^
increment metric when litellm.Router / load balancing logic places a deployment in cool down
N)rC   r   r7  )r^   r  r  r  rA   r  s         ra    increment_deployment_cooled_down1PrometheusLogger.increment_deployment_cooled_down  s'     	++22(BR	

#%r   rK   c                 R    U R                   R                  US9R                  5         g)zQ
Increment metric when logging to a callback fails (e.g., s3_v2, langfuse, etc.)
)rK   N)rJ   r   r7  )r^   rK   s     ra   "increment_callback_logging_failure3PrometheusLogger.increment_callback_logging_failure  s'     	55<<' 	= 	

#%r   providerspendbudget_limitc                 t    U R                   R                  U5      R                  U R                  UUS95        g)z/
Track provider remaining budget in Prometheus

max_budgetr  N)r@   r   rm   _safe_get_remaining_budget)r^   r  r  r  s       ra   track_provider_remaining_budget0PrometheusLogger.track_provider_remaining_budget  s<     	55<<XFJJ++' , 	
r   r  c                 0    Uc  [        S5      $ Uc  U$ X-
  $ )Ninf)float)r^   r  r  s      ra   r  +PrometheusLogger._safe_get_remaining_budget  s(     <=!!r   data_fetch_function.set_metrics_function	data_type)teamskeysc           	        #    SSK Jn  Uc  g SnSnU" XeS9I Sh  vN u  pxUc  [        U5      nX-   S-
  U-  n	U" U5      I Sh  vN   [        SU	S-   5       H#  nU" XeS9I Sh  vN u  pzU" U5      I Sh  vN   M%     g Ne N= N N! [         a/  n[
        R                  " SU S	[        U5       35         SnAgSnAff = f7f)
a2  
Generic method to initialize budget metrics for teams or API keys.

Args:
    data_fetch_function: Function to fetch data with pagination.
    set_metrics_function: Function to set metrics for the fetched data.
    data_type: String representing the type of data ("teams" or "keys") for logging purposes.
r   prisma_clientNr  2   )	page_sizepager  zError initializing z budget metrics: )litellm.proxy.proxy_serverr  r  ranger\   r   rn  r]   )r^   r  r  r  r  r  r  r  total_counttotal_pages_r`   s               ra   _initialize_budget_metrics+PrometheusLogger._initialize_budget_metrics  s      	= 	DI&9#' !D "!$i '2Q69DK 't,,, aq1 3i SS*4000 2! - T0 	$$%i[0A#a&J 	st   CB	 B)B	 BB	 %B&B	 7B8B	  CB	 B	 B	 B	 	
C%B=8C=CCc           	        ^^#    SSK Jm  SSKJm  Tc  [        R
                  " S5        gS[        S[        S[        [        [           [        [           4   4UU4S	 jjnU R                  UU R                  S
S9I Sh  vN   g N7f)zI
Initialize team budget metrics by reusing the generic pagination logic.
r   )get_paginated_teamsr  NzDPrometheus: skipping team metrics initialization, DB not initializedr  r  rd   c                 T   >#    T" TXS9I S h  vN u  p#Uc  [        U5      nX#4$  N7f)N)r  r  r  )r  )r  r  r  r  r  r  s       ra   fetch_teamsEPrometheusLogger._initialize_team_budget_metrics.<locals>.fetch_teams  s>      (;+y( "E "!%j%%"s   (&(r  r  r  r  )1litellm.proxy.management_endpoints.team_endpointsr  r  r  r   rl   intr   r
   r   r   r  _set_team_list_budget_metrics)r^   r  r  r  s     @@ra   _initialize_team_budget_metrics0PrometheusLogger._initialize_team_budget_metrics	  s     	
 	=   V 	&	&"%	&4)*HSM9:	& 	& -- +!%!C!C . 
 	
 	
s   A;B?B Bc           
      @  ^^^#    SSK Jn  SSKJm  SSKJm  SSKJm  Tc  [        R                  " S5        gS[        S	[        S
[        [        U[        [        4      [        [           4   4UUU4S jjnU R!                  UU R"                  SS9I Sh  vN   g N7f)zL
Initialize API key budget metrics by reusing the generic pagination logic.
r   )Union)UI_SESSION_TOKEN_TEAM_ID)_list_key_helperr  NzCPrometheus: skipping key metrics initialization, DB not initializedr  r  rd   c                    >#    T" TUU S S S S TSS S9
I S h  vN nUR                  S/ 5      nUR                  S5      nUc  [        U5      nX44$  N87f)NT)
r  r  sizer$  r  r  key_hashexclude_team_idreturn_full_objectorganization_idr  r  )r   r  )r  r  key_list_responser  r  r  r  r  s        ra   
fetch_keysGPrometheusLogger._initialize_api_key_budget_metrics.<locals>.fetch_keys:  sw      '7+ 8#' $' ! %((4D+//>K"!$i$$!!s   AA9Ar  r  )r   r  litellm.constantsr  ;litellm.proxy.management_endpoints.key_management_endpointsr  r  r  r   rl   r  r   r
   r]   r   r   r  _set_key_list_budget_metrics)r^   r  r  r  r  r  s      @@@ra   "_initialize_api_key_budget_metrics3PrometheusLogger._initialize_api_key_budget_metrics(  s      	!>	
 	=   U 	%	%"%	%4c>123Xc]BC	% 	%* -- *!%!B!B . 
 	
 	
s   BBBBc                   #    SSK Jn  SSKJn  UR                  R
                  nU(       a_  UR                  (       aN  UR                  US9I Sh  vN (       a1   U R                  5       I Sh  vN   UR                  US9I Sh  vN   ggU R                  5       I Sh  vN   g NT N8 N#! UR                  US9I Sh  vN    f = f N(7f)aq  
Handler for initializing remaining budget metrics for all teams to avoid metric discrepancies.

Runs when prometheus logger starts up.

- If redis cache is available, we use the pod lock manager to acquire a lock and initialize the metrics.
    - Ensures only one pod emits the metrics at a time.
- If redis cache is not available, we initialize the metrics directly.
r   )'PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME)proxy_logging_obj)
cronjob_idN)
r	  r  r  r  db_spend_update_writerpod_lock_managerredis_cacheacquire_lock$_initialize_remaining_budget_metricsrelease_lock)r^   r  r  r  s       ra   #initialize_remaining_budget_metrics4PrometheusLogger.initialize_remaining_budget_metricsU  s      	N@,CCTT  0 < <%22B 3   CCEEE*77#J 8    ;;=== F*77#J 8   
 >sf   AC	B$	C	B* -B&.B* 2C	B(C	CC	&B* (C	*C=C >CC	c                    #    [         R                  " S5        U R                  5       I Sh  vN   U R                  5       I Sh  vN   g N N7f)zK
Helper to initialize remaining budget metrics for all teams and API keys.
z%Emitting key, team budget metrics....N)r   rl   r  r  )r^   s    ra   r  5PrometheusLogger._initialize_remaining_budget_metricss  sB      	DE2244455777 	57s!   *AA
AAAAr  c                 n   #    U H+  n[        U[        5      (       d  M  U R                  U5        M-     g7f)z8Helper function to set budget metrics for a list of keysN)ro   r   _set_key_budget_metrics)r^   r  keys      ra   r  -PrometheusLogger._set_key_list_budget_metrics{  s,      C#~..,,S1 s   55r  c                 @   #    U H  nU R                  U5        M     g7f)z9Helper function to set budget metrics for a list of teamsN)_set_team_budget_metrics)r^   r  rN   s      ra   r  .PrometheusLogger._set_team_list_budget_metrics  s     D))$/ s   rN  rO  c                    #    U(       a6  U R                  UU=(       d    SUUUS9I Sh  vN nU R                  U5        gg N7f)z
Set team budget metrics after an LLM API request

- Assemble a LiteLLM_TeamTable object
    - looks up team info from db if not available in metadata
- Set team budget metrics
r   )r  rO   r  r  r
  N)_assemble_team_objectr!  )r^   r"  r#  rN  rO  r
  team_objects          ra   rQ  ;PrometheusLogger._set_team_budget_metrics_after_api_request  sR       $ : :%.4" *+ !; ! K ))+6    'AAAr  rO   c                 *  #    SSK Jn  SSKJnJn  U=(       d    SU-   n	[        UUU	US9n
 U" UUUS9I Sh  vN nU(       a  UR                  U
l
        U
$  N! [         a.  n[        R                  " S[        U5       35        U
s SnA$ SnAff = f7f)z
Assemble a LiteLLM_TeamTable object

for fields not available in metadata, we fetch from db
Fields not available in metadata:
- `budget_reset_at`
r   )get_team_objectr  user_api_key_cache)r  rO   r  r  )r  r  r+  Nz4[Non-Blocking] Prometheus: Error getting team info: )litellm.proxy.auth.auth_checksr)  r  r  r+  r   r\   r   rl   r]   budget_reset_at)r^   r  rO   r  r  r
  r)  r  r+  _total_team_spendr%  	team_infor`   s                ra   r$  &PrometheusLogger._assemble_team_object  s      	CP"Za=8'!#!	

	-+#5 I *3*C*CK'
  	  Fs1vhO 		sC   )BA AA BA 
B"#BBBBBrN   c                    [        UR                  UR                  =(       d    SS9n[        U R	                  SS9US9nU R
                  R                  " S0 UD6R                  U R                  UR                  UR                  S95        UR                  bL  [        U R	                  SS9US9nU R                  R                  " S0 UD6R                  UR                  5        UR                  bZ  [        U R	                  S	S9US9nU R                  R                  " S0 UD6R                  U R                  UR                  S
95        gg)z^
Set team budget metrics for a single team

- Remaining Budget
- Max Budget
- Budget Reset At
r   )rN   rO   r0   r(  r)  r  Nr1   r2   r-  rh   )r,  r  rO   r6  rY   r0   r   rm   r  r  r  r1   r-  r2   %_get_remaining_hours_for_budget_reset)r^   rN   r%  rA  s       ra   r!  )PrometheusLogger._set_team_budget_metrics  s^    ,,"

 +"&"<"<B #= # $	
 	1188C7CGG++??jj , 	
 ??&.&*&@&@ @ 'A ' (	G //66AAEEdooV+.&*&@&@ L 'A ' (	G ;;BBMWMQQ::$($8$8 ;  ,r   c                 l   [        UR                  UR                  =(       d    SS9n[        U R	                  SS9US9nU R
                  R                  " S
0 UD6R                  U R                  UR                  UR                  S95        UR                  bL  [        U R	                  SS9US9nU R                  R                  " S
0 UD6R                  UR                  5        UR                  bC  U R                  R                  " S
0 UD6R                  U R                  UR                  S	95        gg)zS
Set virtual key budget metrics

- Remaining Budget
- Max Budget
- Budget Reset At
r   )r7   r8   r3   r(  r)  r  Nr4   r2  rh   )r,  tokenr  r6  rY   r3   r   rm   r  r  r  r4   r-  r5   r3  )r^   rv  r%  rA  s       ra   r  (PrometheusLogger._set_key_budget_metrics  sG    ,,22+55;
 +"&"<"<E #= # $	
 	44;;FgFJJ++,77'-- , 	
 ''3.&*&@&@ C 'A ' (	G 2299DGDHH!,, ,,8>>EEPPTT::$5$E$E ;  9r   rL  rM  c                    #    U(       a6  U R                  UU=(       d    SUUUS9I S h  vN nU R                  U5        g g  N7f)Nr   )r!  r  rL  rM  r
  )_assemble_key_objectr  )r^   r!  r  r
  rL  rM  rv  s          ra   rP  >PrometheusLogger._set_api_key_budget_metrics_after_api_request*  sU      &*&?&?)#5#;-#+ '@ ' ! (():; !r'  c                 ^  #    SSK Jn  SSKJnJn  U=(       d    SU-   n	[        UUUU	S9n
 U
R                  (       a2  U" U
R                  UUS9I Sh  vN nU(       a  UR                  U
l        U
$  N! [         a-  n[        R                  " S[        U5       35         SnAU
$ SnAff = f7f)z"
Assemble a UserAPIKeyAuth object
r   )get_key_objectr*  )r6  r  r  r  )hashed_tokenr  r+  Nz3[Non-Blocking] Prometheus: Error getting key info: )r,  r<  r  r  r+  r   r6  r-  r\   r   rl   r]   )r^   r!  r  rL  rM  r
  r<  r  r+  _total_key_spendrv  
key_objectr`   s                ra   r9  %PrometheusLogger._assemble_key_object<  s      	BP%Nm;*(%"	
	 &&#1!2!8!8"/'9$ 

 8B8R8R%5 !   	  Ec!fXN  ! 	s@   )B-&A3 A1A3 /B-1A3 3
B*="B%B-%B**B-r-  c                 j    U[         R                  " UR                  5      -
  R                  5       S-  $ )z&
Get remaining hours for budget reset
i  )r   rf  tzinfor  )r^   r-  s     ra   r3  6PrometheusLogger._get_remaining_hours_for_budget_reseta  s.    
 hll?+A+ABB
-/D! 	!r   r9  r:  c                 |    [        U[        5      (       a'  [        U[        5      (       a  X!-
  R                  5       $ g)z
Compute the duration in seconds between two objects.

Returns the duration as a float if both start and end are instances of datetime,
otherwise returns None.
N)ro   r   r  )r^   r9  r:  s      ra   rg  'PrometheusLogger._safe_duration_secondsi  s2     j(++
8X0N0N)88::r   	schedulerc                 P   SSK Jn  SSKJn  [        R
                  R                  [        S9n[        R                  " S[        U5      5        [        U5      S:  aJ  [        [        US   5      n[        R                  " SU-  5        U R                  UR                  SUSS	S
9  gg)z
Initialize budget metrics as a cron job. This job runs every `PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES` minutes.

It emits the current remaining budget metrics for all Keys and Teams.
r   )2PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTESr   )callback_typezfound %s prometheus loggerszNInitializing remaining budget metrics as a cron job executing every %s minutesintervalprometheus_budget_metrics_jobT)minutesr  replace_existingN)r	  rH  "litellm.integrations.custom_loggerr   ri   logging_callback_managerget_custom_loggers_for_typer   r   rl   r  r   add_jobr  )rF  rH  r   prometheus_loggersprometheus_loggers        ra   "initialize_budget_metrics_cron_job3PrometheusLogger.initialize_budget_metrics_cron_jobx  s     	YC ,,HH. I  	 	:C@R<ST!"Q& $%57I!7L M  `DE !EEJ2!%   'r   c                      SSK Jn   SSKJn  SSKJn  S[        R                  ;   a)  SSK JnJ	n  U" 5       nUR                  U5        U " U5      nOU " 5       nUR                  SU5        UR                  S5        g	)
z
Mount the Prometheus metrics endpoint with optional authentication.

Args:
    require_auth (bool, optional): Whether to require authentication for the metrics endpoint.
                                Defaults to False.
r   )make_asgi_app)verbose_proxy_logger)appPROMETHEUS_MULTIPROC_DIR)CollectorRegistrymultiprocessz/metricsz;Starting Prometheus Metrics on /metrics (no authentication)N)rR   rW  litellm._loggingrX  r  rY  osenvironr[  r\  MultiProcessCollectormountrl   )rW  rX  rY  r[  r\  registrymetrics_apps          ra   _mount_metrics_endpoint(PrometheusLogger._mount_metrics_endpoint  sf     	492 &3I(*H..x8'1K'/K 			*k*""I	
r   )%rV   rW   rX   rn   rT   r5   r4   rJ   rC   rI   rE   rG   rB   rD   rH   rF   r.   rL   r*   r+   r/   r?   r@   r#   r'   r3   r6   r:   r<   r0   r>   r(   rQ   r,   r2   r1   r[   N)g      ?)Vr  
__module____qualname____firstlineno__rb   r	   r]   r
   rS   r   rt   r   r   r   r   r   rz   r   rv   r   tupler   r   r{   boolr   rU   r   rY   rB  r   r,  r1  rp   r  r2  r0  r3  r4  rr  r\   r   r  r  rp  r5  staticmethodr  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r   r   r  r  r  r  r  r  r  r   r  rQ  r$  r!  r  rP  r9  r   r3  rg  r   rT  rd  __static_attributes__rh   r   ra   r   r   #   s   Yv*$sDI~*> *XY4 YDU Y.	'	((,S		&	',4 Dd3i<P c 4PS9 *P:"3P:	P:d))04S	)IMc)	)V)#&)7<)	)^	,c 	,DC!#tCy.1DC	DCL
3c 
3d 
35	c.hMT8
"88
 c]8
 sm	8

 %SM8
 }8
  }8
 &c]8
 #8
 +8
t$
}$
 &c]$
 sm	$

 %SM$
 $
 $
LGc]G smG %SM	G
 }G  }G &c]G #G G +G<  sm   %SM   	  
   D@@ }@ sm	@
 %SM@  }@ &c]@ +@D)` (,AA &A *	A
  }AF''-;'RV V|  #RR
 +R Rh $Y $3 $ $,M$',M15,MKT,M\+I$'+I15+IKT+IZCC  C 3-	C
 3-C C2	
	
 	
 		

 	
	
	
 3-	
 3-		

 	
	
	
 3-	
 3-		

 	
  	
  		

$)
9>
	""5/	"2:5/	"		"+%c9U49hsm;S5T+U&UV+ 'S	{IdO'CD+ ?+	+Z
>+
Z><82sN23420>O9P 0
7}7 &c]7 	7
 7 74(( ( 	(
 UO( ( 
(T11f) )V<sm< %SM< 	<
 < E?<$#!#!  #! 	#!
 E?#! #! 
#!J!X !RW !  
%	 6F  > 
 
r   r   r*  r%  tagrd   c                 6   UR                  5       nUR                  5        VVs0 s H  u  pEX@;   d  M  XE_M     nnn[        R                  R                  U;   a  [        SUR                  0SS9US'   UR                  b9  UR                  R                  5        H  u  pu[        U5      nX;   d  M  XVU'   M     UR                  b9  [        UR                  5      n	U	R                  5        H  u  puXp;   d  M  XVU'   M     U  H  nXF;  d  M
  SXd'   M     U$ s  snnf )z
Returns a dictionary of label + values for prometheus.

Ensures end_user param is not sent to prometheus if it is not supported.
user_api_key_end_user_idr   )r   r  rM   N)
model_dumpr   UserAPIKeyLabelNamesEND_USERvaluer   rM   r  r   r  get_custom_labels_from_tags)
r*  r%  rn  	enum_dictr   rt  r   r  sanitized_keycustom_tag_labelss
             ra   r6  r6    s(    &&(I
 &OO--LE) 	-   $$**o=&G68L8LM%'

#
 ))5%<<BBDJC;C@M516.	 E #78H8HI+113JC+',$ 4 ''%)O" ' =s
   DDr   c                    [         R                  nUb  [        U5      S:X  a  0 $ 0 nU H  nUnUR                  S5      (       a  UR	                  SSS5      OUnUR                  S5      nU nU H2  n[        U[        5      (       a  UR                  US5      nOSnUb  M2    O   Uc  M  [        U[        5      (       d  M  XbUR	                  SS5      '   M     U$ )z!
Get custom labels from metadata
Nr   z	metadata.r   r  r  r  )
ri   !custom_prometheus_metadata_labelsr  r.  replacesplitro   rp   r   r]   )r   r  resultr  original_key
keys_partsrt  parts           ra   r-  r-    s     44D|s4yA~	F141L1Lckk+r1-RUYYs^
D%&&		$-}  E3!7!75:<''S12# & Mr   r  configured_tagc                 v   ^^ SSK mSSKJn  U" 5       nUR                  U5      m[	        UU4S jU  5       5      $ )a  
Check if any of the request tags matches a wildcard configured pattern

Args:
    tags: List[str] - The request tags
    configured_tag: str - The configured tag

Returns:
    bool - True if any of the request tags matches the configured tag, False otherwise

e.g.
tags = ["User-Agent: curl/7.68.0", "User-Agent: python-requests/2.28.1", "prod"]
configured_tag = "User-Agent: curl/*"
_tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True

configured_tag = "User-Agent: python-requests/*"
_tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True

configured_tag = "gm"
_tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # False
r   N)PatternMatchRouterc              3   D   >#    U  H  nTR                  TUS 9v   M     g7f))patternstringN)match).0rn  reregex_patterns     ra   	<genexpr>;_tag_matches_wildcard_configured_pattern.<locals>.<genexpr>$	  s     Kdsrxxcx:ds    )r  .litellm.router_utils.pattern_match_deploymentsr  _pattern_to_regexany)r  r  r  pattern_routerr  r  s       @@ra   (_tag_matches_wildcard_configured_patternr  	  s4    0 Q')N"44^DMKdKKKr   c                     SSK Jn  [        R                  nUb  [	        U5      S:X  a  0 $ 0 nU H7  nU" SU 35      nX@;   a  SX5'   M  SU;   a  [        XS9(       a  SX5'   M3  SX5'   M9     U$ )an  
Get custom labels from tags based on admin configuration.

Supports both exact matches and wildcard patterns:
- Exact match: "prod" matches "prod" exactly
- Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0"

Reuses PatternMatchRouter for wildcard pattern matching.

Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise

{
    "tag_User-Agent_curl": "true",
    "tag_User-Agent_python_requests": "false",
    "tag_Environment_prod": "true",
    "tag_Environment_dev": "false",
    "tag_Service_api_gateway_v2": "true",
    "tag_Service_web_app_v1": "false",
}
r   r   tag_truer   )r  r  false)rj   r   ri   custom_prometheus_tagsr  r  )r  r   configured_tagsr}  r  
label_names         ra   ru  ru  '	  s    , V44O#o"6!";	F)4tN;K5LM
 !!'F . %M&
 "(F %! *$ Mr   rf  ))r^  r\  r   r   r   r   r   r   r   r	   r
   r   r   r   r   ri   r]  r   r   rN  r   litellm.proxy._typesr   r   rj   r   r+  r   litellm.utilsr   apscheduler.schedulers.asyncior   r   r]   r,  rp   r6  r-  rk  r  ru  rh   r   ra   <module>r     s    
 
 (    : ; B 3 Q 6 ;?Q"
| Q"
nD ,9,&, 
#, 
	,^d tCH~ @L
s)L%(L	LB0d3i 0DcN 0r   