Error processing request" try=1 err="rpc error: code = Code(500) desc = empty ring

Grafana Loki
1

Hello! I have error when try get log from Grafana:

level=error ts=2022-11-18T16:42:36.297541797Z caller=retry.go:73 org_id=fake msg=“error processing request” try=1 err=“rpc error: code = Code(500) desc = empty ring\n”
ts=2022-11-18T16:42:36.297917582Z caller=spanlogger.go:80 user=fake method=query.Label level=info org_id=fake latency=fast query_type=labels length=1h0m0.001s duration=7.533µs status=500 label= throughput=0B total_bytes=0B total_entries=0

My docker-compose.yml

#version: "3.8"

services:

  init:
    image: grafana/loki:2.6.1
    user: root
    entrypoint:
      - "chown"
      - "10001:10001"
      - "/loki"
    volumes:
      - ./loki:/loki
    networks:
      - loki

  cassandra:
    image: bitnami/cassandra:3.11.11
    restart: always
    ports:
      - "9042:9042"
    volumes:
      - cassandra:/bitnami
    healthcheck:
      test: ["CMD", "cqlsh", "-u", "cassandra", "-p", "cassandra", "-e", "SELECT table_name FROM system_schema.tables WHERE keyspace_name='loki';"]
      interval: 5s
      timeout: 5s
      retries: 10
    environment:
      CASSANDRA_USER: cassandra
      CASSANDRA_PASSWORD: cassandra
      CASSANDRA_CLUSTER_NAME: loki
      CASSANDRA_DATACENTER: dc1
    networks:
      loki:
        aliases:
          - cassandra

  log-generator:
    image: mingrammer/flog
    restart: always
    command:
      - --loop
      - --format=json
      - --number=10
      - --delay=100ms
      - --output=/var/log/generated-logs.txt
      - --overwrite
      - --type=log
    depends_on:
      promtail:
        condition: service_started
    volumes:
      - ./loki/:/var/log/

  promtail:
    image: grafana/promtail:2.6.1
    restart: always
    volumes:
      - ./loki/:/var/log/
      - ./config:/etc/promtail/
    ports:
      - "9080:9080"
    command: -config.file=/etc/promtail/promtail.yaml
    depends_on:
      loki-gateway:
        condition: service_started
      loki-frontend:
        condition: service_started
      loki-read:
        condition: service_started
      loki-write:
        condition: service_started
    networks:
      loki:
        aliases:
          - promtail

  minio:
    image: minio/minio
    restart: always
    entrypoint:
      - sh
      - -euc
      - |
        mkdir -p /data/loki-data && \
        mkdir -p /data/loki-ruler &&
        minio server /data
    environment:
      - MINIO_ACCESS_KEY=loki
      - MINIO_SECRET_KEY=supersecret
      - MINIO_PROMETHEUS_AUTH_TYPE=public
      - MINIO_UPDATE=off
    ports:
      - "9000:9000"
    volumes:
      - ./.data/minio:/data
    networks:
      loki:
        aliases:
          - minio

  loki-gateway:
    image: nginx:1.19
    restart: always
    volumes:
      - ./config/nginx.conf:/etc/nginx/nginx.conf
    ports:
      - "8080:80"
      - "3100"
    networks:
      loki:
        aliases:
          - loki-gateway
    depends_on:
      loki-frontend:
        condition: service_started
      loki-read:
        condition: service_started
      loki-write:
        condition: service_started

  loki-frontend:
    image: grafana/loki:2.6.1
    restart: always
    volumes:
      - ./config:/etc/loki/
    ports:
      - "3100"
    command: "-config.file=/etc/loki/loki.yaml -target=query-frontend -frontend.downstream-url=http://loki-read:3100"
    networks:
      loki:
        aliases:
          - loki-frontend
    deploy:
      mode: replicated
      replicas: 1
    depends_on:
      cassandra:
        condition: service_healthy

  loki-read:
    image: grafana/loki:2.6.1
    restart: always
    volumes:
      - ./config:/etc/loki/
    ports:
      - "3100"
      - "7946"
    command: "-config.file=/etc/loki/loki.yaml -target=read"
    networks:
      loki:
        aliases:
          - loki-read
    deploy:
      mode: replicated
      replicas: 1
    depends_on:
      cassandra:
        condition: service_healthy

  loki-write:
    image: grafana/loki:2.6.1
    restart: always
    volumes:
      - ./config:/etc/loki/
    ports:
      - "3100"
      - "7946"
    command: "-config.file=/etc/loki/loki.yaml -target=write"
    networks:
      loki:
        aliases:
          - loki-write
    deploy:
      mode: replicated
      replicas: 1
    depends_on:
      cassandra:
        condition: service_healthy

  grafana:
    image: grafana/grafana:9.1.6
    restart: always
    ports:
      - "3000:3000"
    environment:
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_DISABLE_LOGIN_FORM: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin"
    volumes:
      - ./config/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yml
      - grafana:/var/lib/grafana
    networks:
      - loki

volumes:
  cassandra:
  grafana:

networks:
  loki:
    driver: bridge

My loki.yaml

auth_enabled: false

#target: ingester, querier, query-scheduler, query-frontend, ruler, table-manager, read, write.
target: table_manager

server:
  http_listen_address: 0.0.0.0
  grpc_listen_address: 0.0.0.0
  http_listen_port: 3100
  http_server_read_timeout: 180s
  http_server_write_timeout: 180s

common:
  path_prefix: /loki/
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

#memberlist:
#  join_members: ["loki-read", "loki-write", "cassandra", "ingester"]
#  dead_node_reclaim_time: 30s
#  gossip_to_dead_nodes_time: 15s
#  left_ingesters_timeout: 30s
#  bind_addr: ['0.0.0.0']
#  bind_port: 7946
#  gossip_interval: 2s

ingester:
  lifecycler:
    join_after: 10s
    observe_period: 5s
    ring:
      replication_factor: 1
      kvstore:
        store: inmemory
    final_sleep: 0s
  chunk_idle_period: 1h
  wal:
    enabled: true
    dir: /loki/wal
  max_chunk_age: 1h
  chunk_target_size: 1048576
  chunk_retain_period: 30s
  max_transfer_retries: 0

schema_config:
  configs:
    - from: 2021-08-19
      store: cassandra
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  cassandra:
    username: cassandra
    password: cassandra
    addresses: cassandra
    consistency: ONE
    auth: true
    keyspace: loki
    timeout: 20s
    connect_timeout: 20s
  filesystem:
    directory: /loki/chunks

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_rate_mb: 500
  ingestion_burst_size_mb: 500
  max_cache_freshness_per_query: '10m'
  max_query_series: 100000
  max_query_parallelism: 16
  split_queries_by_interval: 60m

chunk_store_config:
  max_look_back_period: 720h

table_manager:
  retention_deletes_enabled: true
  retention_period: 720h

query_range:
  align_queries_with_step: true
  max_retries: 10
  cache_results: true
  parallelise_shardable_queries: true

frontend:
  log_queries_longer_than: 5s
  compress_responses: true

query_scheduler:
  max_outstanding_requests_per_tenant: 1024

querier:
  engine:
    max_look_back_period: 720h
  query_ingesters_within: 2h
  query_timeout: 1200s

#compactor:
#  working_directory: /tmp/compactor
#  shared_store: s3

ruler:
  storage:
    type: local
    local:
      directory: /loki/rules
  ring:
    kvstore:
      store: inmemory
  rule_path: /loki/scratch

How fix this issue ?

2

This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.