### Hide pydantic namespace conflict warnings globally ### from __future__ import annotations import warnings warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*") # Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances # This warning can accumulate during streaming and cause memory leaks warnings.filterwarnings( "ignore", message=".*Accessing the.*attribute on the instance is deprecated.*" ) ### INIT VARIABLES ######################### import threading import os from typing import ( Callable, List, Optional, Dict, Union, Any, Literal, get_args, TYPE_CHECKING, Tuple, overload, Type, ) from litellm.types.integrations.datadog import DatadogInitParams from litellm._logging import ( set_verbose, _turn_on_debug, verbose_logger, json_logs, _turn_on_json, log_level, ) import re from litellm.constants import ( DEFAULT_BATCH_SIZE, DEFAULT_FLUSH_INTERVAL_SECONDS, ROUTER_MAX_FALLBACKS, DEFAULT_MAX_RETRIES, DEFAULT_REPLICATE_POLLING_RETRIES, DEFAULT_REPLICATE_POLLING_DELAY_SECONDS, LITELLM_CHAT_PROVIDERS, HUMANLOOP_PROMPT_CACHE_TTL_SECONDS, OPENAI_CHAT_COMPLETION_PARAMS, OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility OPENAI_FINISH_REASONS, OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility openai_compatible_endpoints, openai_compatible_providers, openai_text_completion_compatible_providers, _openai_like_providers, replicate_models, clarifai_models, huggingface_models, empower_models, together_ai_models, baseten_models, WANDB_MODELS, REPEATED_STREAMING_CHUNK_LIMIT, request_timeout, open_ai_embedding_models, cohere_embedding_models, bedrock_embedding_models, known_tokenizer_config, BEDROCK_INVOKE_PROVIDERS_LITERAL, BEDROCK_EMBEDDING_PROVIDERS_LITERAL, BEDROCK_CONVERSE_MODELS, DEFAULT_MAX_TOKENS, DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, ) import httpx import dotenv # register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" if litellm_mode == "DEV": dotenv.load_dotenv() #################################################### if set_verbose: _turn_on_debug() #################################################### ### Callbacks /Logging / Success / Failure Handlers ##### CALLBACK_TYPES = Union[str, Callable, "CustomLogger"] # CustomLogger is lazy-loaded input_callback: List[CALLBACK_TYPES] = [] success_callback: List[CALLBACK_TYPES] = [] failure_callback: List[CALLBACK_TYPES] = [] service_callback: List[CALLBACK_TYPES] = [] # logging_callback_manager is lazy-loaded via __getattr__ _custom_logger_compatible_callbacks_literal = Literal[ "lago", "openmeter", "logfire", "literalai", "dynamic_rate_limiter", "dynamic_rate_limiter_v3", "langsmith", "prometheus", "otel", "datadog", "datadog_llm_observability", "galileo", "braintrust", "arize", "arize_phoenix", "langtrace", "gcs_bucket", "azure_storage", "opik", "argilla", "mlflow", "langfuse", "langfuse_otel", "weave_otel", "pagerduty", "humanloop", "azure_sentinel", "gcs_pubsub", "agentops", "anthropic_cache_control_hook", "generic_api", "resend_email", "sendgrid_email", "smtp_email", "deepeval", "s3_v2", "aws_sqs", "vector_store_pre_call_hook", "dotprompt", "bitbucket", "gitlab", "cloudzero", "focus", "posthog", "levo", ] cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None _known_custom_logger_compatible_callbacks: List = list( get_args(_custom_logger_compatible_callbacks_literal) ) callbacks: List[ Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded ] = [] callback_settings: Dict[str, Dict[str, Any]] = {} initialized_langfuse_clients: int = 0 langfuse_default_tags: Optional[List[str]] = None langsmith_batch_size: Optional[int] = None prometheus_initialize_budget_metrics: Optional[bool] = False require_auth_for_metrics_endpoint: Optional[bool] = False argilla_batch_size: Optional[int] = None datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload. gcs_pub_sub_use_v1: Optional[bool] = ( False # if you want to use v1 gcs pubsub logged payload ) generic_api_use_v1: Optional[bool] = ( False # if you want to use v1 generic api logged payload ) argilla_transformation_object: Optional[Dict[str, Any]] = None _async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. _async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. _async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] post_call_rules: List[Callable] = [] turn_off_message_logging: Optional[bool] = False standard_logging_payload_excluded_fields: Optional[List[str]] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it log_raw_request_response: bool = False redact_messages_in_exceptions: Optional[bool] = False redact_user_api_key_info: Optional[bool] = False filter_invalid_headers: Optional[bool] = False add_user_information_to_llm_headers: Optional[bool] = ( None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers ) store_audit_logs = False # Enterprise feature, allow users to see audit logs ### end of callbacks ############# email: Optional[str] = ( None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) token: Optional[str] = ( None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) retry = True ### AUTH ### api_key: Optional[str] = None openai_key: Optional[str] = None groq_key: Optional[str] = None gigachat_key: Optional[str] = None databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None anthropic_key: Optional[str] = None replicate_key: Optional[str] = None bytez_key: Optional[str] = None cohere_key: Optional[str] = None infinity_key: Optional[str] = None clarifai_key: Optional[str] = None maritalk_key: Optional[str] = None ai21_key: Optional[str] = None ollama_key: Optional[str] = None openrouter_key: Optional[str] = None datarobot_key: Optional[str] = None predibase_key: Optional[str] = None huggingface_key: Optional[str] = None vertex_project: Optional[str] = None vertex_location: Optional[str] = None predibase_tenant_id: Optional[str] = None togetherai_api_key: Optional[str] = None cloudflare_api_key: Optional[str] = None vercel_ai_gateway_key: Optional[str] = None baseten_key: Optional[str] = None llama_api_key: Optional[str] = None aleph_alpha_key: Optional[str] = None nlp_cloud_key: Optional[str] = None novita_api_key: Optional[str] = None snowflake_key: Optional[str] = None gradient_ai_api_key: Optional[str] = None nebius_key: Optional[str] = None wandb_key: Optional[str] = None heroku_key: Optional[str] = None cometapi_key: Optional[str] = None ovhcloud_key: Optional[str] = None lemonade_key: Optional[str] = None sap_service_key: Optional[str] = None amazon_nova_api_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], } use_litellm_proxy: bool = ( False # when True, requests will be sent to the specified litellm proxy endpoint ) use_client: bool = False ssl_verify: Union[str, bool] = True ssl_security_level: Optional[str] = None ssl_certificate: Optional[str] = None ssl_ecdh_curve: Optional[str] = ( None # Set to 'X25519' to disable PQC and improve performance ) disable_streaming_logging: bool = False disable_token_counter: bool = False disable_add_transform_inline_image_block: bool = False disable_add_user_agent_to_request_tags: bool = False disable_anthropic_gemini_context_caching_transform: bool = False extra_spend_tag_headers: Optional[List[str]] = None in_memory_llm_clients_cache: "LLMClientCache" safe_memory_mode: bool = False enable_azure_ad_token_refresh: Optional[bool] = False # Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy proxy_auth: Optional[Any] = None ### DEFAULT AZURE API VERSION ### AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest ### DEFAULT WATSONX API VERSION ### WATSONX_DEFAULT_API_VERSION = "2024-03-13" ### COHERE EMBEDDINGS DEFAULT TYPE ### COHERE_DEFAULT_EMBEDDING_INPUT_TYPE: "COHERE_EMBEDDING_INPUT_TYPES" = "search_document" ### CREDENTIALS ### credential_list: List["CredentialItem"] = [] ### GUARDRAILS ### llamaguard_model_name: Optional[str] = None openai_moderations_model_name: Optional[str] = None presidio_ad_hoc_recognizers: Optional[str] = None google_moderation_confidence_threshold: Optional[float] = None llamaguard_unsafe_content_categories: Optional[str] = None blocked_user_list: Optional[Union[str, List]] = None banned_keywords_list: Optional[Union[str, List]] = None llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all" guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False reasoning_auto_summary: bool = False ### PROMPTS #### from litellm.types.prompts.init_prompts import PromptSpec prompt_name_config_map: Dict[str, PromptSpec] = {} ################## ### PREVIEW FEATURES ### enable_preview_features: bool = False return_response_headers: bool = ( False # get response headers from LLM Api providers - example x-remaining-requests, ) enable_json_schema_validation: bool = False #################### logging: bool = True enable_loadbalancing_on_batch_endpoints: Optional[bool] = None enable_caching_on_provider_specific_optional_params: bool = ( False # feature-flag for caching on optional params - e.g. 'top_k' ) caching: bool = ( False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) caching_with_models: bool = ( False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648 ) cache: Optional["Cache"] = ( None # cache object <- use this - https://docs.litellm.ai/docs/caching ) default_in_memory_ttl: Optional[float] = None default_redis_ttl: Optional[float] = None default_redis_batch_cache_expiry: Optional[float] = None model_alias_map: Dict[str, str] = {} model_group_settings: Optional["ModelGroupSettings"] = None max_budget: float = 0.0 # set the max budget across all providers budget_duration: Optional[str] = ( None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"). ) default_soft_budget: float = ( DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0 ) forward_traceparent_to_llm_provider: bool = False _current_cost = 0.0 # private variable, used if max budget is set error_logs: Dict = {} add_function_to_prompt: bool = ( False # if function calling not supported by api, append function call details to system prompt ) client_session: Optional[httpx.Client] = None aclient_session: Optional[httpx.AsyncClient] = None model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' model_cost_map_url: str = os.getenv( "LITELLM_MODEL_COST_MAP_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json", ) anthropic_beta_headers_url: str = os.getenv( "LITELLM_ANTHROPIC_BETA_HEADERS_URL", "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json", ) suppress_debug_info = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None datadog_params: Optional[Union[DatadogInitParams, Dict]] = None aws_sqs_callback_params: Optional[Dict] = None generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None upperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None key_generation_settings: Optional["StandardKeyGenerationConfig"] = None default_internal_user_params: Optional[Dict] = None default_team_params: Optional[Union[DefaultTeamSSOParams, Dict]] = None default_team_settings: Optional[List] = None max_user_budget: Optional[float] = None default_max_internal_user_budget: Optional[float] = None max_internal_user_budget: Optional[float] = None max_ui_session_budget: Optional[float] = 0.25 # $0.25 USD budgets for UI Chat sessions internal_user_budget_duration: Optional[str] = None tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None max_end_user_budget: Optional[float] = None max_end_user_budget_id: Optional[str] = None disable_end_user_cost_tracking: Optional[bool] = None disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None custom_prometheus_metadata_labels: List[str] = [] custom_prometheus_tags: List[str] = [] prometheus_metrics_config: Optional[List] = None disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) disable_copilot_system_to_assistant: bool = ( False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. ) public_mcp_servers: Optional[List[str]] = None public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {} #### REQUEST PRIORITIZATION ####### priority_reservation: Optional[ Dict[str, Union[float, "PriorityReservationDict"]] ] = None # priority_reservation_settings is lazy-loaded via __getattr__ # Only declare for type checking - at runtime __getattr__ handles it if TYPE_CHECKING: priority_reservation_settings: Optional["PriorityReservationSettings"] = None ######## Networking Settings ######## use_aiohttp_transport: bool = ( True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead. ) aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = ( False # When False, aiohttp will respect HTTP(S)_PROXY env vars ) force_ipv4: bool = ( False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. ) ####### STOP SEQUENCE LIMIT ####### disable_stop_sequence_limit: bool = False # when True, stop sequence limit is disabled #### RETRIES #### num_retries: Optional[int] = None # per model endpoint max_fallbacks: Optional[int] = None default_fallbacks: Optional[List] = None fallbacks: Optional[List] = None context_window_fallbacks: Optional[List] = None content_policy_fallbacks: Optional[List] = None allowed_fails: int = 3 allow_dynamic_callback_disabling: bool = True num_retries_per_request: Optional[int] = ( None # for the request overall (incl. fallbacks + model retries) ) ####### SECRET MANAGERS ##################### secret_manager_client: Optional[Any] = ( None # list of instantiated key management clients - e.g. azure kv, infisical, etc. ) _google_kms_resource_name: Optional[str] = None _key_management_system: Optional["KeyManagementSystem"] = None # Note: KeyManagementSettings must be eagerly imported because _key_management_settings # is accessed during import time in secret_managers/main.py # We'll import it after the lazy import system is set up # We can't define it here because KeyManagementSettings is lazy-loaded #### PII MASKING #### output_parse_pii: bool = False ############################################# from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map model_cost = get_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = ( {} ) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = ( {} ) # Provider-specific or global cost margins. Examples: # Percentage: {"openai": 0.10} = 10% margin # Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request # Global: {"global": 0.05} = 5% global margin on all providers # Combined: {"vertex_ai": {"percentage": 0.08, "fixed_amount": 0.0005}} custom_prompt_dict: Dict[str, dict] = {} check_provider_endpoint = False ####### THREAD-SPECIFIC DATA #################### class MyLocal(threading.local): def __init__(self): self.user = "Hello World" _thread_context = MyLocal() def identify(event_details): # Store user in thread local data if "user" in event_details: _thread_context.user = event_details["user"] ####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc. api_base: Optional[str] = None headers = None api_version: Optional[str] = None organization = None project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set open_ai_chat_completion_models: Set = set() open_ai_text_completion_models: Set = set() cohere_models: Set = set() cohere_chat_models: Set = set() mistral_chat_models: Set = set() text_completion_codestral_models: Set = set() anthropic_models: Set = set() openrouter_models: Set = set() datarobot_models: Set = set() vertex_language_models: Set = set() vertex_vision_models: Set = set() vertex_chat_models: Set = set() vertex_code_chat_models: Set = set() vertex_ai_image_models: Set = set() vertex_ai_video_models: Set = set() vertex_text_models: Set = set() vertex_code_text_models: Set = set() vertex_embedding_models: Set = set() vertex_anthropic_models: Set = set() vertex_llama3_models: Set = set() vertex_deepseek_models: Set = set() vertex_ai_ai21_models: Set = set() vertex_mistral_models: Set = set() vertex_openai_models: Set = set() vertex_minimax_models: Set = set() vertex_moonshot_models: Set = set() vertex_zai_models: Set = set() ai21_models: Set = set() ai21_chat_models: Set = set() nlp_cloud_models: Set = set() aleph_alpha_models: Set = set() bedrock_models: Set = set() bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS) fal_ai_models: Set = set() fireworks_ai_models: Set = set() fireworks_ai_embedding_models: Set = set() deepinfra_models: Set = set() perplexity_models: Set = set() watsonx_models: Set = set() gemini_models: Set = set() xai_models: Set = set() zai_models: Set = set() deepseek_models: Set = set() runwayml_models: Set = set() azure_ai_models: Set = set() jina_ai_models: Set = set() voyage_models: Set = set() infinity_models: Set = set() heroku_models: Set = set() databricks_models: Set = set() cloudflare_models: Set = set() codestral_models: Set = set() friendliai_models: Set = set() featherless_ai_models: Set = set() palm_models: Set = set() groq_models: Set = set() azure_models: Set = set() azure_anthropic_models: Set = set() azure_text_models: Set = set() anyscale_models: Set = set() cerebras_models: Set = set() galadriel_models: Set = set() nvidia_nim_models: Set = set() sambanova_models: Set = set() sambanova_embedding_models: Set = set() novita_models: Set = set() assemblyai_models: Set = set() snowflake_models: Set = set() gradient_ai_models: Set = set() llama_models: Set = set() nscale_models: Set = set() nebius_models: Set = set() nebius_embedding_models: Set = set() aiml_models: Set = set() deepgram_models: Set = set() elevenlabs_models: Set = set() dashscope_models: Set = set() moonshot_models: Set = set() publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() hyperbolic_models: Set = set() recraft_models: Set = set() cometapi_models: Set = set() oci_models: Set = set() vercel_ai_gateway_models: Set = set() volcengine_models: Set = set() wandb_models: Set = set(WANDB_MODELS) ovhcloud_models: Set = set() ovhcloud_embedding_models: Set = set() lemonade_models: Set = set() docker_model_runner_models: Set = set() amazon_nova_models: Set = set() stability_models: Set = set() github_copilot_models: Set = set() chatgpt_models: Set = set() minimax_models: Set = set() aws_polly_models: Set = set() gigachat_models: Set = set() llamagate_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: """ Excludes keys with the pattern 'bedrock//'. These are in the model_prices_and_context_window.json file for pricing purposes only. Args: key (str): A key to filter. Returns: bool: True if the key matches the Bedrock pattern, False otherwise. """ # Regex to match 'bedrock//' bedrock_pattern = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$") if "month-commitment" in key: return True is_match = bedrock_pattern.match(key) return is_match is not None def is_openai_finetune_model(key: str) -> bool: """ Excludes model cost keys with the pattern 'ft:'. These are in the model_prices_and_context_window.json file for pricing purposes only. Args: key (str): A key to filter. Returns: bool: True if the key matches the OpenAI finetune pattern, False otherwise. """ return key.startswith("ft:") and not key.count(":") > 1 def add_known_models(): for key, value in model_cost.items(): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model( key ): open_ai_chat_completion_models.add(key) elif value.get("litellm_provider") == "text-completion-openai": open_ai_text_completion_models.add(key) elif value.get("litellm_provider") == "azure_text": azure_text_models.add(key) elif value.get("litellm_provider") == "cohere": cohere_models.add(key) elif value.get("litellm_provider") == "cohere_chat": cohere_chat_models.add(key) elif value.get("litellm_provider") == "mistral": mistral_chat_models.add(key) elif value.get("litellm_provider") == "anthropic": anthropic_models.add(key) elif value.get("litellm_provider") == "empower": empower_models.add(key) elif value.get("litellm_provider") == "openrouter": openrouter_models.add(key) elif value.get("litellm_provider") == "vercel_ai_gateway": vercel_ai_gateway_models.add(key) elif value.get("litellm_provider") == "datarobot": datarobot_models.add(key) elif value.get("litellm_provider") == "vertex_ai-text-models": vertex_text_models.add(key) elif value.get("litellm_provider") == "vertex_ai-code-text-models": vertex_code_text_models.add(key) elif value.get("litellm_provider") == "vertex_ai-language-models": vertex_language_models.add(key) elif value.get("litellm_provider") == "vertex_ai-vision-models": vertex_vision_models.add(key) elif value.get("litellm_provider") == "vertex_ai-chat-models": vertex_chat_models.add(key) elif value.get("litellm_provider") == "vertex_ai-code-chat-models": vertex_code_chat_models.add(key) elif value.get("litellm_provider") == "vertex_ai-embedding-models": vertex_embedding_models.add(key) elif value.get("litellm_provider") == "vertex_ai-anthropic_models": key = key.replace("vertex_ai/", "") vertex_anthropic_models.add(key) elif value.get("litellm_provider") == "vertex_ai-llama_models": key = key.replace("vertex_ai/", "") vertex_llama3_models.add(key) elif value.get("litellm_provider") == "vertex_ai-deepseek_models": key = key.replace("vertex_ai/", "") vertex_deepseek_models.add(key) elif value.get("litellm_provider") == "vertex_ai-mistral_models": key = key.replace("vertex_ai/", "") vertex_mistral_models.add(key) elif value.get("litellm_provider") == "vertex_ai-ai21_models": key = key.replace("vertex_ai/", "") vertex_ai_ai21_models.add(key) elif value.get("litellm_provider") == "vertex_ai-image-models": key = key.replace("vertex_ai/", "") vertex_ai_image_models.add(key) elif value.get("litellm_provider") == "vertex_ai-video-models": key = key.replace("vertex_ai/", "") vertex_ai_video_models.add(key) elif value.get("litellm_provider") == "vertex_ai-openai_models": key = key.replace("vertex_ai/", "") vertex_openai_models.add(key) elif value.get("litellm_provider") == "vertex_ai-minimax_models": key = key.replace("vertex_ai/", "") vertex_minimax_models.add(key) elif value.get("litellm_provider") == "vertex_ai-moonshot_models": key = key.replace("vertex_ai/", "") vertex_moonshot_models.add(key) elif value.get("litellm_provider") == "vertex_ai-zai_models": key = key.replace("vertex_ai/", "") vertex_zai_models.add(key) elif value.get("litellm_provider") == "ai21": if value.get("mode") == "chat": ai21_chat_models.add(key) else: ai21_models.add(key) elif value.get("litellm_provider") == "nlp_cloud": nlp_cloud_models.add(key) elif value.get("litellm_provider") == "aleph_alpha": aleph_alpha_models.add(key) elif value.get( "litellm_provider" ) == "bedrock" and not is_bedrock_pricing_only_model(key): bedrock_models.add(key) elif value.get("litellm_provider") == "bedrock_converse": bedrock_converse_models.add(key) elif value.get("litellm_provider") == "deepinfra": deepinfra_models.add(key) elif value.get("litellm_provider") == "perplexity": perplexity_models.add(key) elif value.get("litellm_provider") == "watsonx": watsonx_models.add(key) elif value.get("litellm_provider") == "gemini": gemini_models.add(key) elif value.get("litellm_provider") == "fireworks_ai": # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) elif value.get("litellm_provider") == "fireworks_ai-embedding-models": # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key: fireworks_ai_embedding_models.add(key) elif value.get("litellm_provider") == "text-completion-codestral": text_completion_codestral_models.add(key) elif value.get("litellm_provider") == "xai": xai_models.add(key) elif value.get("litellm_provider") == "zai": zai_models.add(key) elif value.get("litellm_provider") == "fal_ai": fal_ai_models.add(key) elif value.get("litellm_provider") == "deepseek": deepseek_models.add(key) elif value.get("litellm_provider") == "runwayml": runwayml_models.add(key) elif value.get("litellm_provider") == "meta_llama": llama_models.add(key) elif value.get("litellm_provider") == "nscale": nscale_models.add(key) elif value.get("litellm_provider") == "azure_ai": azure_ai_models.add(key) elif value.get("litellm_provider") == "voyage": voyage_models.add(key) elif value.get("litellm_provider") == "infinity": infinity_models.add(key) elif value.get("litellm_provider") == "databricks": databricks_models.add(key) elif value.get("litellm_provider") == "cloudflare": cloudflare_models.add(key) elif value.get("litellm_provider") == "codestral": codestral_models.add(key) elif value.get("litellm_provider") == "friendliai": friendliai_models.add(key) elif value.get("litellm_provider") == "palm": palm_models.add(key) elif value.get("litellm_provider") == "groq": groq_models.add(key) elif value.get("litellm_provider") == "azure": azure_models.add(key) elif value.get("litellm_provider") == "azure_anthropic": azure_anthropic_models.add(key) elif value.get("litellm_provider") == "anyscale": anyscale_models.add(key) elif value.get("litellm_provider") == "cerebras": cerebras_models.add(key) elif value.get("litellm_provider") == "galadriel": galadriel_models.add(key) elif value.get("litellm_provider") == "nvidia_nim": nvidia_nim_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": sambanova_embedding_models.add(key) elif value.get("litellm_provider") == "novita": novita_models.add(key) elif value.get("litellm_provider") == "nebius-chat-models": nebius_models.add(key) elif value.get("litellm_provider") == "nebius-embedding-models": nebius_embedding_models.add(key) elif value.get("litellm_provider") == "aiml": aiml_models.add(key) elif value.get("litellm_provider") == "assemblyai": assemblyai_models.add(key) elif value.get("litellm_provider") == "jina_ai": jina_ai_models.add(key) elif value.get("litellm_provider") == "snowflake": snowflake_models.add(key) elif value.get("litellm_provider") == "gradient_ai": gradient_ai_models.add(key) elif value.get("litellm_provider") == "featherless_ai": featherless_ai_models.add(key) elif value.get("litellm_provider") == "deepgram": deepgram_models.add(key) elif value.get("litellm_provider") == "elevenlabs": elevenlabs_models.add(key) elif value.get("litellm_provider") == "heroku": heroku_models.add(key) elif value.get("litellm_provider") == "dashscope": dashscope_models.add(key) elif value.get("litellm_provider") == "moonshot": moonshot_models.add(key) elif value.get("litellm_provider") == "publicai": publicai_models.add(key) elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "morph": morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "recraft": recraft_models.add(key) elif value.get("litellm_provider") == "cometapi": cometapi_models.add(key) elif value.get("litellm_provider") == "oci": oci_models.add(key) elif value.get("litellm_provider") == "volcengine": volcengine_models.add(key) elif value.get("litellm_provider") == "wandb": wandb_models.add(key) elif value.get("litellm_provider") == "ovhcloud": ovhcloud_models.add(key) elif value.get("litellm_provider") == "ovhcloud-embedding-models": ovhcloud_embedding_models.add(key) elif value.get("litellm_provider") == "lemonade": lemonade_models.add(key) elif value.get("litellm_provider") == "docker_model_runner": docker_model_runner_models.add(key) elif value.get("litellm_provider") == "amazon_nova": amazon_nova_models.add(key) elif value.get("litellm_provider") == "stability": stability_models.add(key) elif value.get("litellm_provider") == "github_copilot": github_copilot_models.add(key) elif value.get("litellm_provider") == "chatgpt": chatgpt_models.add(key) elif value.get("litellm_provider") == "minimax": minimax_models.add(key) elif value.get("litellm_provider") == "aws_polly": aws_polly_models.add(key) elif value.get("litellm_provider") == "gigachat": gigachat_models.add(key) elif value.get("litellm_provider") == "llamagate": llamagate_models.add(key) add_known_models() # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary # this is maintained for Exception Mapping # used for Cost Tracking & Token counting # https://azure.microsoft.com/en-in/pricing/details/cognitive-services/openai-service/ # Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting azure_llms = { "gpt-35-turbo": "azure/gpt-35-turbo", "gpt-35-turbo-16k": "azure/gpt-35-turbo-16k", "gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct", "azure/gpt-41": "gpt-4.1", "azure/gpt-41-mini": "gpt-4.1-mini", "azure/gpt-41-nano": "gpt-4.1-nano", } azure_embedding_models = { "ada": "azure/ada", } petals_models = [ "petals-team/StableBeluga2", ] ollama_models = ["llama2"] maritalk_models = ["maritalk"] model_list = list( open_ai_chat_completion_models | open_ai_text_completion_models | cohere_models | cohere_chat_models | anthropic_models | set(replicate_models) | openrouter_models | datarobot_models | set(huggingface_models) | vertex_chat_models | vertex_text_models | ai21_models | ai21_chat_models | set(together_ai_models) | set(baseten_models) | aleph_alpha_models | nlp_cloud_models | set(ollama_models) | bedrock_models | deepinfra_models | perplexity_models | set(maritalk_models) | runwayml_models | vertex_language_models | watsonx_models | gemini_models | text_completion_codestral_models | xai_models | zai_models | fal_ai_models | deepseek_models | azure_ai_models | voyage_models | infinity_models | databricks_models | cloudflare_models | codestral_models | friendliai_models | palm_models | groq_models | azure_models | azure_anthropic_models | anyscale_models | cerebras_models | galadriel_models | nvidia_nim_models | sambanova_models | azure_text_models | novita_models | assemblyai_models | jina_ai_models | snowflake_models | gradient_ai_models | llama_models | featherless_ai_models | nscale_models | deepgram_models | elevenlabs_models | dashscope_models | moonshot_models | publicai_models | v0_models | morph_models | lambda_ai_models | recraft_models | cometapi_models | oci_models | heroku_models | vercel_ai_gateway_models | volcengine_models | wandb_models | ovhcloud_models | lemonade_models | docker_model_runner_models | set(clarifai_models) ) model_list_set = set(model_list) # provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time models_by_provider: dict = { "openai": open_ai_chat_completion_models | open_ai_text_completion_models, "text-completion-openai": open_ai_text_completion_models, "cohere": cohere_models | cohere_chat_models, "cohere_chat": cohere_chat_models, "anthropic": anthropic_models, "replicate": replicate_models, "huggingface": huggingface_models, "together_ai": together_ai_models, "baseten": baseten_models, "openrouter": openrouter_models, "vercel_ai_gateway": vercel_ai_gateway_models, "datarobot": datarobot_models, "vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models | vertex_minimax_models | vertex_moonshot_models | vertex_zai_models, "ai21": ai21_models, "bedrock": bedrock_models | bedrock_converse_models, "petals": petals_models, "ollama": ollama_models, "ollama_chat": ollama_models, "deepinfra": deepinfra_models, "perplexity": perplexity_models, "maritalk": maritalk_models, "watsonx": watsonx_models, "gemini": gemini_models, "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models, "aleph_alpha": aleph_alpha_models, "text-completion-codestral": text_completion_codestral_models, "xai": xai_models, "zai": zai_models, "fal_ai": fal_ai_models, "deepseek": deepseek_models, "runwayml": runwayml_models, "mistral": mistral_chat_models, "azure_ai": azure_ai_models, "voyage": voyage_models, "infinity": infinity_models, "databricks": databricks_models, "cloudflare": cloudflare_models, "codestral": codestral_models, "nlp_cloud": nlp_cloud_models, "friendliai": friendliai_models, "palm": palm_models, "groq": groq_models, "azure": azure_models | azure_text_models, "azure_anthropic": azure_anthropic_models, "azure_text": azure_text_models, "anyscale": anyscale_models, "cerebras": cerebras_models, "galadriel": galadriel_models, "nvidia_nim": nvidia_nim_models, "sambanova": sambanova_models | sambanova_embedding_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, "aiml": aiml_models, "assemblyai": assemblyai_models, "jina_ai": jina_ai_models, "snowflake": snowflake_models, "gradient_ai": gradient_ai_models, "meta_llama": llama_models, "nscale": nscale_models, "featherless_ai": featherless_ai_models, "deepgram": deepgram_models, "elevenlabs": elevenlabs_models, "heroku": heroku_models, "dashscope": dashscope_models, "moonshot": moonshot_models, "publicai": publicai_models, "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, "hyperbolic": hyperbolic_models, "recraft": recraft_models, "cometapi": cometapi_models, "oci": oci_models, "volcengine": volcengine_models, "wandb": wandb_models, "ovhcloud": ovhcloud_models | ovhcloud_embedding_models, "lemonade": lemonade_models, "clarifai": clarifai_models, "amazon_nova": amazon_nova_models, "stability": stability_models, "github_copilot": github_copilot_models, "chatgpt": chatgpt_models, "minimax": minimax_models, "aws_polly": aws_polly_models, "gigachat": gigachat_models, "llamagate": llamagate_models, } # mapping for those models which have larger equivalents longer_context_model_fallback_dict: dict = { # openai chat completion models "gpt-3.5-turbo": "gpt-3.5-turbo-16k", "gpt-3.5-turbo-0301": "gpt-3.5-turbo-16k-0301", "gpt-3.5-turbo-0613": "gpt-3.5-turbo-16k-0613", "gpt-4": "gpt-4-32k", "gpt-4-0314": "gpt-4-32k-0314", "gpt-4-0613": "gpt-4-32k-0613", # anthropic "claude-instant-1": "claude-2", "claude-instant-1.2": "claude-2", # vertexai "chat-bison": "chat-bison-32k", "chat-bison@001": "chat-bison-32k", "codechat-bison": "codechat-bison-32k", "codechat-bison@001": "codechat-bison-32k", # openrouter "openrouter/openai/gpt-3.5-turbo": "openrouter/openai/gpt-3.5-turbo-16k", "openrouter/anthropic/claude-instant-v1": "openrouter/anthropic/claude-2", } ####### EMBEDDING MODELS ################### all_embedding_models = ( open_ai_embedding_models | set(cohere_embedding_models) | set(bedrock_embedding_models) | vertex_embedding_models | fireworks_ai_embedding_models | nebius_embedding_models | sambanova_embedding_models | ovhcloud_embedding_models ) ####### IMAGE GENERATION MODELS ################### openai_image_generation_models = ["dall-e-2", "dall-e-3"] ####### VIDEO GENERATION MODELS ################### openai_video_generation_models = ["sora-2"] # timeout is lazy-loaded via __getattr__ # get_llm_provider is lazy-loaded via __getattr__ # remove_index_from_tool_calls is lazy-loaded via __getattr__ # Import KeyManagementSettings here (before utils import) because _key_management_settings # is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) from litellm.types.secret_managers.main import KeyManagementSettings _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time from .llms.custom_llm import CustomLLM from .llms.anthropic.common_utils import AnthropicModelInfo from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config from .llms.deprecated_providers.palm import ( PalmConfig, ) # here to prevent breaking changes from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig from .llms.gemini.common_utils import GeminiModelInfo from .llms.vertex_ai.vertex_embeddings.transformation import ( VertexAITextEmbeddingConfig, ) vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig() from .llms.bedrock.embed.amazon_titan_v2_transformation import ( AmazonTitanV2Config, ) from .llms.topaz.common_utils import TopazModelInfo # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) # All remaining configs are now lazy loaded - see _lazy_imports_registry.py # Import LlmProviders here (before main import) because it's imported during import time # in multiple places including openai.py (via main import) from litellm.types.utils import LlmProviders ## Lazy loading this is not straightforward, will leave it here for now. from .main import * # type: ignore # Skills API from .skills.main import ( create_skill, acreate_skill, list_skills, alist_skills, get_skill, aget_skill, delete_skill, adelete_skill, ) from .integrations import * from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients from .exceptions import ( AuthenticationError, InvalidRequestError, BadRequestError, ImageFetchError, NotFoundError, PermissionDeniedError, RateLimitError, ServiceUnavailableError, BadGatewayError, OpenAIError, ContextWindowExceededError, ContentPolicyViolationError, BudgetExceededError, APIError, Timeout, APIConnectionError, UnsupportedParamsError, APIResponseValidationError, UnprocessableEntityError, InternalServerError, JSONSchemaValidationError, LITELLM_EXCEPTION_TYPES, MockException, ) from .budget_manager import BudgetManager from .proxy.proxy_cli import run_server from .router import Router from .assistants.main import * from .batches.main import * from .images.main import * from .videos.main import * from .batch_completion.main import * # type: ignore from .rerank_api.main import * from .llms.anthropic.experimental_pass_through.messages.handler import * from .responses.main import * # Interactions API is available as litellm.interactions module # Usage: litellm.interactions.create(), litellm.interactions.get(), etc. from . import interactions from .skills.main import ( create_skill, acreate_skill, list_skills, alist_skills, get_skill, aget_skill, delete_skill, adelete_skill, ) from .containers.main import * from .ocr.main import * from .rag.main import * from .search.main import * from .realtime_api.main import _arealtime from .fine_tuning.main import * from .files.main import * from .vector_store_files.main import ( acreate as avector_store_file_create, adelete as avector_store_file_delete, alist as avector_store_file_list, aretrieve as avector_store_file_retrieve, aretrieve_content as avector_store_file_content, aupdate as avector_store_file_update, create as vector_store_file_create, delete as vector_store_file_delete, list as vector_store_file_list, retrieve as vector_store_file_retrieve, retrieve_content as vector_store_file_content, update as vector_store_file_update, ) from .scheduler import * ### ADAPTERS ### from .types.adapter import AdapterItem import litellm.anthropic_interface as anthropic adapters: List[AdapterItem] = [] ### Vector Store Registry ### from .vector_stores.vector_store_registry import ( VectorStoreRegistry, VectorStoreIndexRegistry, ) vector_store_registry: Optional[VectorStoreRegistry] = None vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None ### RAG ### from . import rag ### CUSTOM LLMs ### from .types.llms.custom_llm import CustomLLMItem custom_provider_map: List[CustomLLMItem] = [] _custom_providers: List[str] = ( [] ) # internal helper util, used to track names of custom providers disable_hf_tokenizer_download: Optional[bool] = ( None # disable huggingface tokenizer download. Defaults to openai clk100 ) global_disable_no_log_param: bool = False ### CLI UTILITIES ### from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key ### PASSTHROUGH ### from .passthrough import allm_passthrough_route, llm_passthrough_route from .google_genai import agenerate_content ### GLOBAL CONFIG ### global_bitbucket_config: Optional[Dict[str, Any]] = None def set_global_bitbucket_config(config: Dict[str, Any]) -> None: """Set global BitBucket configuration for prompt management.""" global global_bitbucket_config global_bitbucket_config = config ### GLOBAL CONFIG ### global_gitlab_config: Optional[Dict[str, Any]] = None def set_global_gitlab_config(config: Dict[str, Any]) -> None: """Set global BitBucket configuration for prompt management.""" global global_gitlab_config global_gitlab_config = config # Lazy loading system for heavy modules to reduce initial import time and memory usage if TYPE_CHECKING: from litellm.types.utils import ModelInfo as _ModelInfoType from litellm.types.utils import PriorityReservationSettings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching.caching import Cache # Type stubs for lazy-loaded configs to help mypy from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig from .llms.maritalk import MaritalkConfig as MaritalkConfig from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig from .llms.triton.completion.transformation import TritonConfig as TritonConfig from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import AmazonMoonshotConfig as AmazonMoonshotConfig from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig from .llms.a2a.chat.transformation import A2AConfig as A2AConfig from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig from .llms.mistral.chat.transformation import MistralConfig as MistralConfig from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig from .llms.perplexity.responses.transformation import PerplexityResponsesConfig as PerplexityResponsesConfig from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig # Type stubs for lazy-loaded config instances openaiOSeriesConfig: OpenAIOSeriesConfig openAIGPTConfig: OpenAIGPTConfig openAIGPTAudioConfig: OpenAIGPTAudioConfig openAIGPT5Config: OpenAIGPT5Config nvidiaNimConfig: NvidiaNimConfig nvidiaNimEmbeddingConfig: NvidiaNimEmbeddingConfig # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] DeepSeekChatConfig: Type[_DeepSeekChatConfig] GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig] GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig] AzureOpenAIO1Config: Type[_AzureOpenAIO1Config] PerplexityChatConfig: Type[_PerplexityChatConfig] NscaleConfig: Type[_NscaleConfig] IBMWatsonXChatConfig: Type[_IBMWatsonXChatConfig] IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig] LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig] DeepInfraConfig: Type[_DeepInfraConfig] LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig from .llms.hosted_vllm.embedding.transformation import HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig from .llms.wandb.chat.transformation import WandbConfig as WandbConfig from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( BudgetConfig, CredentialItem, PriorityReservationDict, StandardKeyGenerationConfig, ) from litellm.types.guardrails import GuardrailItem from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, LiteLLM_UpperboundKeyGenerateParams, ) # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] completion_cost: Callable[..., float] response_cost_calculator: Any modify_integration: Any # Utils functions - type stubs for truly lazy loaded functions only # (functions NOT imported via "from .main import *") get_response_string: Callable[..., str] supports_function_calling: Callable[..., bool] supports_web_search: Callable[..., bool] supports_url_context: Callable[..., bool] supports_response_schema: Callable[..., bool] supports_parallel_function_calling: Callable[..., bool] supports_vision: Callable[..., bool] supports_audio_input: Callable[..., bool] supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] acreate: Callable[..., Any] get_max_tokens: Callable[..., int] get_model_info: Callable[..., _ModelInfoType] register_prompt_template: Callable[..., None] validate_environment: Callable[..., dict] check_valid_key: Callable[..., bool] register_model: Callable[..., None] encode: Callable[..., list] decode: Callable[..., str] _calculate_retry_after: Callable[..., float] _should_retry: Callable[..., bool] get_supported_openai_params: Callable[..., Optional[list]] get_api_base: Callable[..., Optional[str]] get_first_chars_messages: Callable[..., str] get_provider_fields: Callable[..., List] get_valid_models: Callable[..., list] remove_index_from_tool_calls: Callable[..., None] # Response types - truly lazy loaded only (not in main.py or elsewhere) ModelResponseListIterator: Type[Any] # HTTP handler singletons (created lazily via __getattr__ at runtime) module_level_aclient: AsyncHTTPHandler module_level_client: HTTPHandler # Bedrock tool name mappings instance (lazy-loaded) from litellm.caching.caching import InMemoryCache bedrock_tool_name_mappings: InMemoryCache # Azure exception class (lazy-loaded) from litellm.llms.azure.common_utils import AzureOpenAIError # Secret manager types (lazy-loaded) from litellm.types.secret_managers.main import ( KeyManagementSystem, KeyManagementSettings, # Not lazy-loaded - needed for _key_management_settings initialization ) # Custom logger class (lazy-loaded) from litellm.integrations.custom_logger import CustomLogger # Datadog LLM observability params (lazy-loaded) from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams # Logging callback manager class and instance (lazy-loaded) from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager logging_callback_manager: LoggingCallbackManager # provider_list is lazy-loaded from litellm.types.utils import LlmProviders provider_list: List[Union[LlmProviders, str]] # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block # Track if async client cleanup has been registered (for lazy loading) _async_client_cleanup_registered = False # Eager loading for backwards compatibility with VCR and other HTTP recording tools # When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time # For now, this only affects encoding (tiktoken) as it was the only reported issue # See: https://github.com/BerriAI/litellm/issues/18659 # This ensures encoding is initialized before VCR starts recording HTTP requests if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"): # Load encoding at import time (pre-#18070 behavior) # This ensures encoding is initialized before VCR starts recording from .main import encoding def __getattr__(name: str) -> Any: """Lazy import handler with cached registry for improved performance.""" global _async_client_cleanup_registered # Register async client cleanup on first access (only once) if not _async_client_cleanup_registered: from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup register_async_client_cleanup() _async_client_cleanup_registered = True # Use cached registry from _lazy_imports instead of importing tuples every time from ._lazy_imports import _get_lazy_import_registry registry = _get_lazy_import_registry() # Check if name is in registry and call the cached handler function if name in registry: handler_func = registry[name] return handler_func(name) # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "encoding" not in _globals: from .main import encoding as _encoding _globals["encoding"] = _encoding return _globals["encoding"] # Lazy load bedrock_tool_name_mappings instance if name == "bedrock_tool_name_mappings": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "bedrock_tool_name_mappings" not in _globals: from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings return _globals["bedrock_tool_name_mappings"] # Lazy load AzureOpenAIError exception class if name == "AzureOpenAIError": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "AzureOpenAIError" not in _globals: from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError _globals["AzureOpenAIError"] = _AzureOpenAIError return _globals["AzureOpenAIError"] # Lazy load openaiOSeriesConfig instance if name == "openaiOSeriesConfig": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() if "openaiOSeriesConfig" not in _globals: # Import the config class and instantiate it config_class = __getattr__("OpenAIOSeriesConfig") _globals["openaiOSeriesConfig"] = config_class() return _globals["openaiOSeriesConfig"] # Lazy load other config instances _config_instances = { "openAIGPTConfig": "OpenAIGPTConfig", "openAIGPTAudioConfig": "OpenAIGPTAudioConfig", "openAIGPT5Config": "OpenAIGPT5Config", "nvidiaNimConfig": "NvidiaNimConfig", "nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig", } if name in _config_instances: from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() if name not in _globals: # Import the config class and instantiate it config_class = __getattr__(_config_instances[name]) _globals[name] = config_class() return _globals[name] # Handle OpenAIO1Config alias if name == "OpenAIO1Config": return __getattr__("OpenAIOSeriesConfig") # Lazy load provider_list if name == "provider_list": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "provider_list" not in _globals: # LlmProviders is eagerly imported above, so we can import it directly from litellm.types.utils import LlmProviders _globals["provider_list"] = list(LlmProviders) return _globals["provider_list"] # Lazy load priority_reservation_settings instance if name == "priority_reservation_settings": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "priority_reservation_settings" not in _globals: # Import the class and instantiate it PriorityReservationSettings = __getattr__("PriorityReservationSettings") _globals["priority_reservation_settings"] = PriorityReservationSettings() return _globals["priority_reservation_settings"] # Lazy load logging_callback_manager instance if name == "logging_callback_manager": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "logging_callback_manager" not in _globals: # Import the class and instantiate it LoggingCallbackManager = __getattr__("LoggingCallbackManager") _globals["logging_callback_manager"] = LoggingCallbackManager() return _globals["logging_callback_manager"] # Lazy load _service_logger module if name == "_service_logger": from ._lazy_imports import _get_litellm_globals _globals = _get_litellm_globals() # Check if already cached if "_service_logger" not in _globals: # Import the module lazily import litellm._service_logger _globals["_service_logger"] = litellm._service_logger return _globals["_service_logger"] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") # ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time