litellm-poc/litellm_configs/config.yaml at main · automat-it/litellm-poc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
credential_list:
  - credential_name: aws_creds
    credential_values:
      aws_region_name: os.environ/AWS_REGION
    credential_info:
      description: "AWS credentials to reach Amazon Bedrock Models"

model_list:
  - model_name: nova-lite
    litellm_params:
      model: bedrock/us.amazon.nova-lite-v1:0
      litellm_credential_name: aws_creds
      # rpm: 4
      # tpm: 10000
      weight: 3

  - model_name: nova-pro-us
    litellm_params:
      model: bedrock/us.amazon.nova-pro-v1:0
      litellm_credential_name: aws_creds
      weight: 7
      aws_region_name: us-central-1
    # Overriding default price per token if needed
    # model_info:
    #   input_cost_per_token: 0.000006
    #   output_cost_per_token: 0.00003
    #   cache_creation_input_token_cost: 0.0000075
    #   cache_read_input_token_cost: 0.0000006

  - model_name: nova-pro-eu
    litellm_params:
      model: bedrock/eu.amazon.nova-pro-v1:0
      litellm_credential_name: aws_creds
      # rpm: 400
      # tpm: 300000
      weight: 1
      aws_region_name: eu-central-1


# router_settings ==============================================================
router_settings:
  fallbacks: [
    {"nova-pro-eu": ["nova-pro-us", "nova-lite"]},
    {"nova-pro-us": ["nova-lite", "nova-pro-eu"]}
  ]
  routing_strategy: simple-shuffle
  # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"

  # todo need to setup Redis DB if multiple deployments
  # redis_host: <your redis host>                # set this when using multiple litellm proxy deployments, load balancing state stored in redis
  # redis_password: <your redis password>  # os.environ/<YOUR-ENV-VAR>
  # redis_port: 1992

# END OF router_settings =======================================================

# general_settings =============================================================
general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/LITELLM_DATABASE_URL
  background_health_checks: True
  health_check_interval: 300
  # If running LiteLLM on VPC, gracefully handle DB unavailability
  # https://docs.litellm.ai/docs/proxy/prod
  # allow_requests_on_db_unavailable: True

  # Disable spend_logs & error_logs if not using the LiteLLM UI
  # https://docs.litellm.ai/docs/proxy/prod#6-disable-spend_logs--error_logs-if-not-using-the-litellm-ui
  # disable_spend_logs: True    # Disable writing spend logs to DB
  # disable_error_logs: True    # Disable writing error logs to DB

  # You can enable model health checks being run in the background,
  # to prevent each model from being queried too frequently via /health
  # https://docs.litellm.ai/docs/proxy/health#background-health-checks
  # background_health_checks: True # enable background health checks
  # health_check_interval: 300 # frequency of background health checks

# END OF general_settings ======================================================


# litellm_settings =============================================================

litellm_settings:
  # Use 'langfuse_otel' for v3 compatibility and better performance
  callbacks: ["langfuse_otel"]

  # Optional: capture errors specifically in Langfuse
  failure_callback: ["langfuse_otel"]

  # These tags help filter your traces inside the Langfuse UI
  langfuse_default_tags: [
    "cache_hit",
    "proxy_base_url",
    "user_api_key_alias",
    "semantic-similarity"
  ]
# END OF litellm_settings ======================================================