Promptail HTTP status 429

Grafana Loki
1

promtail.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: deviceLog
  static_configs:
  - targets:
      - localhost
    labels:
      job: localserver
      __path__: /var/log/localserver/002*/*.log
  - targets:
      - localhost
    labels:
      job: bob
      __path__: /var/log/bob/002*/*.log
  - targets:
      - localhost
    labels:
      job: media
      __path__: /var/log/media/002*/*.log
  - targets:
      - localhost
    labels:
      job: test
      __path__: /var/log/test/*.log

- job_name: webLog
  static_configs:
  - targets:
      - localhost
    labels:
      job: image
      __path__: /var/log/image/*.log
  - targets:
      - localhost
    labels:
      job: bob-soultion
      __path__: /var/log/bob-soultion/*.log
  - targets:
      - localhost
    labels:
      job: customer
      __path__: /var/log/customer/*.log

loki.yaml

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

  grpc_server_max_recv_msg_size: 1572864000
  grpc_server_max_send_msg_size: 1572864000

limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  retention_period: 360h
  max_query_series: 1000
  max_query_parallelism: 2 # Maximum number of queries that will be scheduled in parallel by the frontend.
  max_query_lookback: 24h


  enforce_metric_name: false
  max_cache_freshness_per_query: 10m
  split_queries_by_interval: 15m
  # for big logs tune
  per_stream_rate_limit: 512M
  per_stream_rate_limit_burst: 1024M
  cardinality_limit: 200000
  ingestion_burst_size_mb: 1000
  ingestion_rate_mb: 10000
  max_entries_limit_per_query: 1000000
  max_label_value_length: 20480
  max_label_name_length: 10240
  max_label_names_per_series: 300
common:
  instance_addr: 127.0.0.1
  path_prefix: /tmp/loki
  storage:
    filesystem:
      chunks_directory: /tmp/loki/chunks
      rules_directory: /tmp/loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 10000

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

table_manager:
  retention_deletes_enabled: true
  retention_period: 24h


compactor:
  working_directory: /tmp/loki/retention
  shared_store: filesystem
  compaction_interval: 10m
  retention_enabled: true
  retention_delete_delay: 10s
  retention_delete_worker_count: 150

ruler:
  storage:
    type: local
    local:
      directory: /loki/rules
  rule_path: /loki/rules-temp
  alertmanager_url: http://192.168.152.150:9093
  ring:
    kvstore:
      store: inmemory
  enable_api: true
  enable_alertmanager_v2: true

promtail error:

level=warn ts=2023-04-28T08:14:43.443396458Z caller=client.go:419 component=client host=loki:3100 msg="error sending batch, will retry" status=429 tenant= error="server returned HTTP status 429 Too Many Requests (429): Maximum active stream limit exceeded, reduce the number of active streams (reduce labels or reduce label values), or contact your Loki administrator to see if the limit can be increased

How should this be resolved

2

The error gives a good hint: Maximum active stream limit exceeded. I would recommend you to take a look at the limits configuration (Grafana Loki configuration parameters | Grafana Loki documentation) and adjust the stream related settings to see if it helps.

3

Thank you. We have debugged some related configurations but still reported the same error, which has not been resolved yet

1 Like
4

This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.