[HA mode] Different results between two requests

Hello :slight_smile:

I am deploying a loki in HA with currently two nodes (without Kubernetes).

Overall it works fine.

However, when I make requests to find logs, the number of logs returned differs half the time.

For example, query 1 on a fixed range hits instance 1 and returns me 430 records.

I relaunch the same query which hits instance 2 and returns 230 records this time.

Being in the migration phase, I have another loki (non HA) somewhere else that receives the same traffic as my new HA loki.

When I make this same request, I receive 660 records, the sum of the two previous requests (230 + 430) on the loki HA!

Do you have any idea what could be happening?

For information:

  • I use a s3 bucket to store indexes
  • Each loki instance is running in target=all
  • Each Loki instance is contacted through an nginx reverse proxy
  • I have upstream of each reverse proxy, a load balancer provided by my cloud provider which returns in round robin to my nginx servers

Thx for your help !!

My config:

target: all
http_prefix: ""
ballast_bytes: 0
use_buffered_logger: true
use_sync_logger: true
server:
  http_listen_network: tcp
  http_listen_address: ""
  http_listen_port: 3100
  http_listen_conn_limit: 0
  grpc_listen_network: tcp
  grpc_listen_address: ""
  grpc_listen_port: 9095
  grpc_listen_conn_limit: 0
  tls_cipher_suites: ""
  tls_min_version: ""
  http_tls_config:
    cert_file: ""
    key_file: ""
    client_auth_type: ""
    client_ca_file: ""
  grpc_tls_config:
    cert_file: ""
    key_file: ""
    client_auth_type: ""
    client_ca_file: ""
  register_instrumentation: true
  graceful_shutdown_timeout: 30s
  http_server_read_timeout: 30s
  http_server_write_timeout: 30s
  http_server_idle_timeout: 2m0s
  grpc_server_max_recv_msg_size: 4194304
  grpc_server_max_send_msg_size: 4194304
  grpc_server_max_concurrent_streams: 100
  grpc_server_max_connection_idle: 2562047h47m16.854775807s
  grpc_server_max_connection_age: 2562047h47m16.854775807s
  grpc_server_max_connection_age_grace: 2562047h47m16.854775807s
  grpc_server_keepalive_time: 2h0m0s
  grpc_server_keepalive_timeout: 20s
  grpc_server_min_time_between_pings: 10s
  grpc_server_ping_without_stream_allowed: true
  log_format: logfmt
  log_level: error
  log_source_ips_enabled: false
  log_source_ips_header: ""
  log_source_ips_regex: ""
  log_request_at_info_level_enabled: false
  http_path_prefix: ""
internal_server:
  http_listen_network: tcp
  http_listen_address: localhost
  http_listen_port: 3101
  http_listen_conn_limit: 0
  grpc_listen_network: ""
  grpc_listen_address: ""
  grpc_listen_port: 0
  grpc_listen_conn_limit: 0
  tls_cipher_suites: ""
  tls_min_version: ""
  http_tls_config:
    cert_file: ""
    key_file: ""
    client_auth_type: ""
    client_ca_file: ""
  grpc_tls_config:
    cert_file: ""
    key_file: ""
    client_auth_type: ""
    client_ca_file: ""
  register_instrumentation: false
  graceful_shutdown_timeout: 30s
  http_server_read_timeout: 30s
  http_server_write_timeout: 30s
  http_server_idle_timeout: 2m0s
  grpc_server_max_recv_msg_size: 0
  grpc_server_max_send_msg_size: 0
  grpc_server_max_concurrent_streams: 0
  grpc_server_max_connection_idle: 0s
  grpc_server_max_connection_age: 0s
  grpc_server_max_connection_age_grace: 0s
  grpc_server_keepalive_time: 0s
  grpc_server_keepalive_timeout: 0s
  grpc_server_min_time_between_pings: 0s
  grpc_server_ping_without_stream_allowed: false
  log_format: ""
  log_level: ""
  log_source_ips_enabled: false
  log_source_ips_header: ""
  log_source_ips_regex: ""
  log_request_at_info_level_enabled: false
  http_path_prefix: ""
  enable: false
distributor:
  ring:
    kvstore:
      store: memberlist
      prefix: collectors/
      consul:
        host: localhost:8500
        acl_token: ""
        http_client_timeout: 20s
        consistent_reads: false
        watch_rate_limit: 1
        watch_burst_size: 1
        cas_retry_delay: 1s
      etcd:
        endpoints: []
        dial_timeout: 10s
        max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
        username: ""
        password: ""
      multi:
        primary: ""
        secondary: ""
        mirror_enabled: false
        mirror_timeout: 2s
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    instance_id: loki-instance-0
    instance_interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    instance_port: 0
    instance_addr: 127.0.0.1
  rate_store:
    max_request_parallelism: 200
    stream_rate_update_interval: 1s
    ingester_request_timeout: 500ms
querier:
  tail_max_duration: 1h0m0s
  query_ingesters_within: 3h0m0s
  engine:
    timeout: 5m0s
    max_look_back_period: 30s
  max_concurrent: 10
  query_store_only: false
  query_ingester_only: false
  multi_tenant_queries_enabled: false
  query_timeout: 0s
  per_request_limits_enabled: false
query_scheduler:
  max_outstanding_requests_per_tenant: 4096
  querier_forget_delay: 0s
  grpc_client_config:
    max_recv_msg_size: 104857600
    max_send_msg_size: 104857600
    grpc_compression: ""
    rate_limit: 0
    rate_limit_burst: 0
    backoff_on_ratelimits: false
    backoff_config:
      min_period: 100ms
      max_period: 10s
      max_retries: 10
    tls_enabled: false
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
  use_scheduler_ring: true
  scheduler_ring:
    kvstore:
      store: memberlist
      prefix: collectors/
      consul:
        host: localhost:8500
        acl_token: ""
        http_client_timeout: 20s
        consistent_reads: false
        watch_rate_limit: 1
        watch_burst_size: 1
        cas_retry_delay: 1s
      etcd:
        endpoints: []
        dial_timeout: 10s
        max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
        username: ""
        password: ""
      multi:
        primary: ""
        secondary: ""
        mirror_enabled: false
        mirror_timeout: 2s
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    tokens_file_path: ""
    zone_awareness_enabled: false
    instance_id: loki-instance-0
    instance_interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    instance_port: 0
    instance_addr: 127.0.0.1
    instance_availability_zone: ""
frontend:
  log_queries_longer_than: 5s
  max_body_size: 10485760
  query_stats_enabled: false
  max_outstanding_per_tenant: 4096
  querier_forget_delay: 0s
  scheduler_address: ""
  scheduler_dns_lookup_period: 10s
  scheduler_worker_concurrency: 5
  grpc_client_config:
    max_recv_msg_size: 104857600
    max_send_msg_size: 104857600
    grpc_compression: ""
    rate_limit: 0
    rate_limit_burst: 0
    backoff_on_ratelimits: false
    backoff_config:
      min_period: 100ms
      max_period: 10s
      max_retries: 10
    tls_enabled: false
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
  graceful_shutdown_timeout: 5m0s
  instance_interface_names:
  - ens2
  - ens4
  - docker0
  - br-a369ef4d6f79
  - lo
  address: ""
  port: 0
  compress_responses: true
  downstream_url: ""
  tail_proxy_url: ""
  tail_tls_config:
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
query_range:
  split_queries_by_interval: 0s
  align_queries_with_step: true
  results_cache:
    cache:
      enable_fifocache: false
      default_validity: 1h0m0s
      background:
        writeback_goroutines: 10
        writeback_buffer: 10000
      memcached:
        expiration: 0s
        batch_size: 1024
        parallelism: 100
      memcached_client:
        host: ""
        service: memcached
        addresses: ""
        timeout: 100ms
        max_idle_conns: 16
        max_item_size: 0
        update_interval: 1m0s
        consistent_hash: true
        circuit_breaker_consecutive_failures: 10
        circuit_breaker_timeout: 10s
        circuit_breaker_interval: 10s
      redis:
        endpoint: ""
        master_name: ""
        timeout: 500ms
        expiration: 0s
        db: 0
        pool_size: 0
        username: ""
        password: ""
        tls_enabled: false
        tls_insecure_skip_verify: false
        idle_timeout: 0s
        max_connection_age: 0s
      embedded_cache:
        enabled: true
        max_size_mb: 100
        ttl: 24h0m0s
      fifocache:
        max_size_bytes: 1GB
        max_size_items: 0
        ttl: 1h0m0s
        validity: 0s
        size: 0
        purgeinterval: 0s
      prefix: frontend.
      async_cache_write_back_concurrency: 16
      async_cache_write_back_buffer_size: 500
    compression: ""
  cache_results: true
  max_retries: 5
  parallelise_shardable_queries: false
  forward_headers_list: []
ruler:
  external_url: ""
  ruler_client:
    max_recv_msg_size: 104857600
    max_send_msg_size: 104857600
    grpc_compression: ""
    rate_limit: 0
    rate_limit_burst: 0
    backoff_on_ratelimits: false
    backoff_config:
      min_period: 100ms
      max_period: 10s
      max_retries: 10
    tls_enabled: false
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
  evaluation_interval: 1m0s
  poll_interval: 1m0s
  storage:
    type: ""
    azure:
      environment: AzureGlobal
      account_name: ""
      account_key: ""
      container_name: loki
      endpoint_suffix: ""
      use_managed_identity: false
      use_federated_token: false
      user_assigned_id: ""
      use_service_principal: false
      client_id: ""
      client_secret: ""
      tenant_id: ""
      chunk_delimiter: '-'
      download_buffer_size: 512000
      upload_buffer_size: 256000
      upload_buffer_count: 1
      request_timeout: 30s
      max_retries: 5
      min_retry_delay: 10ms
      max_retry_delay: 500ms
    alibabacloud:
      bucket: ""
      endpoint: ""
      access_key_id: ""
      secret_access_key: ""
    gcs:
      bucket_name: ""
      service_account: ""
      chunk_buffer_size: 0
      request_timeout: 0s
      enable_opencensus: true
      enable_http2: true
    s3:
      s3: ""
      s3forcepathstyle: false
      bucketnames: ""
      endpoint: ""
      region: ""
      access_key_id: ""
      secret_access_key: ""
      session_token: ""
      insecure: false
      sse_encryption: false
      http_config:
        idle_conn_timeout: 1m30s
        response_header_timeout: 0s
        insecure_skip_verify: false
        ca_file: ""
      signature_version: v4
      storage_class: STANDARD
      sse:
        type: ""
        kms_key_id: ""
        kms_encryption_context: ""
      backoff_config:
        min_period: 100ms
        max_period: 3s
        max_retries: 5
    bos:
      bucket_name: ""
      endpoint: bj.bcebos.com
      access_key_id: ""
      secret_access_key: ""
    swift:
      auth_version: 0
      auth_url: ""
      internal: false
      username: ""
      user_domain_name: ""
      user_domain_id: ""
      user_id: ""
      password: ""
      domain_id: ""
      domain_name: ""
      project_id: ""
      project_name: ""
      project_domain_id: ""
      project_domain_name: ""
      region_name: ""
      container_name: ""
      max_retries: 3
      connect_timeout: 10s
      request_timeout: 5s
    local:
      directory: ""
  rule_path: /loki/rules-temp
  alertmanager_url: ""
  enable_alertmanager_discovery: false
  alertmanager_refresh_interval: 1m0s
  enable_alertmanager_v2: false
  notification_queue_capacity: 10000
  notification_timeout: 10s
  alertmanager_client:
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
    basic_auth_username: ""
    basic_auth_password: ""
    type: Bearer
  for_outage_tolerance: 1h0m0s
  for_grace_period: 10m0s
  resend_delay: 1m0s
  enable_sharding: false
  sharding_strategy: default
  sharding_algo: by-group
  search_pending_for: 5m0s
  ring:
    kvstore:
      store: memberlist
      prefix: collectors/
      consul:
        host: localhost:8500
        acl_token: ""
        http_client_timeout: 20s
        consistent_reads: false
        watch_rate_limit: 1
        watch_burst_size: 1
        cas_retry_delay: 1s
      etcd:
        endpoints: []
        dial_timeout: 10s
        max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
        username: ""
        password: ""
      multi:
        primary: ""
        secondary: ""
        mirror_enabled: false
        mirror_timeout: 2s
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    instance_id: loki-instance-0
    instance_interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    instance_port: 0
    instance_addr: 127.0.0.1
    num_tokens: 128
  flush_period: 1m0s
  enable_api: true
  enabled_tenants: ""
  disabled_tenants: ""
  query_stats_enabled: false
  disable_rule_group_label: false
  wal:
    tenant: ""
    name: ""
    remotewrite: []
    dir: ruler-wal
    truncate_frequency: 1h0m0s
    min_age: 5m0s
    max_age: 4h0m0s
  wal_cleaner:
    min_age: 12h0m0s
    deprecatedperiod: 0s
  remote_write:
    enabled: false
    config_refresh_period: 10s
ingester_client:
  pool_config:
    client_cleanup_period: 15s
    health_check_ingesters: true
    remote_timeout: 1s
  remote_timeout: 5s
  grpc_client_config:
    max_recv_msg_size: 104857600
    max_send_msg_size: 104857600
    grpc_compression: ""
    rate_limit: 0
    rate_limit_burst: 0
    backoff_on_ratelimits: false
    backoff_config:
      min_period: 100ms
      max_period: 10s
      max_retries: 10
    tls_enabled: false
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
ingester:
  lifecycler:
    ring:
      kvstore:
        store: memberlist
        prefix: collectors/
        consul:
          host: localhost:8500
          acl_token: ""
          http_client_timeout: 20s
          consistent_reads: false
          watch_rate_limit: 1
          watch_burst_size: 1
          cas_retry_delay: 1s
        etcd:
          endpoints: []
          dial_timeout: 10s
          max_retries: 10
          tls_enabled: false
          tls_cert_path: ""
          tls_key_path: ""
          tls_ca_path: ""
          tls_server_name: ""
          tls_insecure_skip_verify: false
          tls_cipher_suites: ""
          tls_min_version: ""
          username: ""
          password: ""
        multi:
          primary: ""
          secondary: ""
          mirror_enabled: false
          mirror_timeout: 2s
      heartbeat_timeout: 1m0s
      replication_factor: 1
      zone_awareness_enabled: false
      excluded_zones: ""
    num_tokens: 128
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    observe_period: 0s
    join_after: 0s
    min_ready_duration: 15s
    interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    final_sleep: 0s
    tokens_file_path: ""
    availability_zone: ""
    unregister_on_shutdown: true
    readiness_check_ring_health: true
    address: 127.0.0.1
    port: 0
    id: loki-instance-0
  concurrent_flushes: 32
  flush_check_period: 30s
  flush_op_timeout: 10m0s
  chunk_retain_period: 1m0s
  chunk_idle_period: 30m0s
  chunk_block_size: 262144
  chunk_target_size: 1572864
  chunk_encoding: snappy
  max_chunk_age: 2h0m0s
  autoforget_unhealthy: false
  sync_period: 0s
  sync_min_utilization: 0
  max_returned_stream_errors: 10
  query_store_max_look_back_period: 2h41m0s
  wal:
    enabled: true
    dir: /loki/wal
    checkpoint_duration: 5m0s
    flush_on_shutdown: false
    replay_memory_ceiling: 4294967296
  index_shards: 32
  max_dropped_streams: 10
index_gateway:
  mode: simple
  ring:
    kvstore:
      store: memberlist
      prefix: collectors/
      consul:
        host: localhost:8500
        acl_token: ""
        http_client_timeout: 20s
        consistent_reads: false
        watch_rate_limit: 1
        watch_burst_size: 1
        cas_retry_delay: 1s
      etcd:
        endpoints: []
        dial_timeout: 10s
        max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
        username: ""
        password: ""
      multi:
        primary: ""
        secondary: ""
        mirror_enabled: false
        mirror_timeout: 2s
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    tokens_file_path: ""
    zone_awareness_enabled: false
    instance_id: loki-instance-0
    instance_interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    instance_port: 0
    instance_addr: 127.0.0.1
    instance_availability_zone: ""
    replication_factor: 1
storage_config:
  alibabacloud:
    bucket: ""
    endpoint: ""
    access_key_id: ""
    secret_access_key: ""
  aws:
    dynamodb:
      dynamodb_url: ""
      api_limit: 2
      throttle_limit: 10
      metrics:
        url: ""
        target_queue_length: 100000
        scale_up_factor: 1.3
        ignore_throttle_below: 1
        queue_length_query: sum(avg_over_time(cortex_ingester_flush_queue_length{job="cortex/ingester"}[2m]))
        write_throttle_query: sum(rate(cortex_dynamo_throttled_total{operation="DynamoDB.BatchWriteItem"}[1m]))
          by (table) > 0
        write_usage_query: sum(rate(cortex_dynamo_consumed_capacity_total{operation="DynamoDB.BatchWriteItem"}[15m]))
          by (table) > 0
        read_usage_query: sum(rate(cortex_dynamo_consumed_capacity_total{operation="DynamoDB.QueryPages"}[1h]))
          by (table) > 0
        read_error_query: sum(increase(cortex_dynamo_failures_total{operation="DynamoDB.QueryPages",error="ProvisionedThroughputExceededException"}[1m]))
          by (table) > 0
      chunk_gang_size: 10
      chunk_get_max_parallelism: 32
      backoff_config:
        min_period: 100ms
        max_period: 50s
        max_retries: 20
      kms_key_id: ""
    s3: s3://######":######@s3.fr-par.wwwww.cloud/my-storage
    s3forcepathstyle: true
    bucketnames: ""
    endpoint: ""
    region: fr-par
    access_key_id: ""
    secret_access_key: ""
    session_token: ""
    insecure: false
    sse_encryption: false
    http_config:
      idle_conn_timeout: 1m30s
      response_header_timeout: 0s
      insecure_skip_verify: false
      ca_file: ""
    signature_version: v4
    storage_class: STANDARD
    sse:
      type: ""
      kms_key_id: ""
      kms_encryption_context: ""
    backoff_config:
      min_period: 100ms
      max_period: 3s
      max_retries: 5
  azure:
    environment: AzureGlobal
    account_name: ""
    account_key: ""
    container_name: loki
    endpoint_suffix: ""
    use_managed_identity: false
    use_federated_token: false
    user_assigned_id: ""
    use_service_principal: false
    client_id: ""
    client_secret: ""
    tenant_id: ""
    chunk_delimiter: '-'
    download_buffer_size: 512000
    upload_buffer_size: 256000
    upload_buffer_count: 1
    request_timeout: 30s
    max_retries: 5
    min_retry_delay: 10ms
    max_retry_delay: 500ms
  bos:
    bucket_name: ""
    endpoint: bj.bcebos.com
    access_key_id: ""
    secret_access_key: ""
  bigtable:
    project: ""
    instance: ""
    grpc_client_config:
      max_recv_msg_size: 104857600
      max_send_msg_size: 104857600
      grpc_compression: ""
      rate_limit: 0
      rate_limit_burst: 0
      backoff_on_ratelimits: false
      backoff_config:
        min_period: 100ms
        max_period: 10s
        max_retries: 10
      tls_enabled: true
      tls_cert_path: ""
      tls_key_path: ""
      tls_ca_path: ""
      tls_server_name: ""
      tls_insecure_skip_verify: false
      tls_cipher_suites: ""
      tls_min_version: ""
    table_cache_enabled: true
    table_cache_expiration: 30m0s
  gcs:
    bucket_name: ""
    service_account: ""
    chunk_buffer_size: 0
    request_timeout: 0s
    enable_opencensus: true
    enable_http2: true
  cassandra:
    addresses: ""
    port: 9042
    keyspace: ""
    consistency: QUORUM
    replication_factor: 3
    disable_initial_host_lookup: false
    SSL: false
    host_verification: true
    host_selection_policy: round-robin
    CA_path: ""
    tls_cert_path: ""
    tls_key_path: ""
    auth: false
    username: ""
    password: ""
    password_file: ""
    custom_authenticators: []
    timeout: 2s
    connect_timeout: 5s
    reconnect_interval: 1s
    max_retries: 0
    retry_max_backoff: 10s
    retry_min_backoff: 100ms
    query_concurrency: 0
    num_connections: 2
    convict_hosts_on_failure: true
    table_options: ""
  boltdb:
    directory: ""
  filesystem:
    directory: ""
  swift:
    auth_version: 0
    auth_url: ""
    internal: false
    username: ""
    user_domain_name: ""
    user_domain_id: ""
    user_id: ""
    password: ""
    domain_id: ""
    domain_name: ""
    project_id: ""
    project_name: ""
    project_domain_id: ""
    project_domain_name: ""
    region_name: ""
    container_name: ""
    max_retries: 3
    connect_timeout: 10s
    request_timeout: 5s
  grpc_store: {}
  hedging:
    at: 0s
    up_to: 2
    max_per_second: 5
  named_stores:
    aws: {}
    azure: {}
    bos: {}
    filesystem: {}
    gcs: {}
    alibabacloud: {}
    swift: {}
  index_cache_validity: 5m0s
  index_queries_cache_config:
    enable_fifocache: false
    default_validity: 1h0m0s
    background:
      writeback_goroutines: 10
      writeback_buffer: 10000
    memcached:
      expiration: 0s
      batch_size: 1024
      parallelism: 100
    memcached_client:
      host: ""
      service: memcached
      addresses: ""
      timeout: 100ms
      max_idle_conns: 16
      max_item_size: 0
      update_interval: 1m0s
      consistent_hash: true
      circuit_breaker_consecutive_failures: 10
      circuit_breaker_timeout: 10s
      circuit_breaker_interval: 10s
    redis:
      endpoint: ""
      master_name: ""
      timeout: 500ms
      expiration: 0s
      db: 0
      pool_size: 0
      username: ""
      password: ""
      tls_enabled: false
      tls_insecure_skip_verify: false
      idle_timeout: 0s
      max_connection_age: 0s
    embedded_cache:
      max_size_mb: 100
      ttl: 1h0m0s
    fifocache:
      max_size_bytes: 1GB
      max_size_items: 0
      ttl: 1h0m0s
      validity: 0s
      size: 0
      purgeinterval: 0s
    prefix: store.index-cache-read.
    async_cache_write_back_concurrency: 16
    async_cache_write_back_buffer_size: 500
  disable_broad_index_queries: false
  max_parallel_get_chunk: 150
  max_chunk_batch_size: 50
  boltdb_shipper:
    active_index_directory: /loki/index
    shared_store: s3
    shared_store_key_prefix: index/
    cache_location: /loki/boltdb-cache
    cache_ttl: 24h0m0s
    resync_interval: 5m0s
    query_ready_num_days: 0
    index_gateway_client:
      grpc_client_config:
        max_recv_msg_size: 104857600
        max_send_msg_size: 104857600
        grpc_compression: ""
        rate_limit: 0
        rate_limit_burst: 0
        backoff_on_ratelimits: false
        backoff_config:
          min_period: 100ms
          max_period: 10s
          max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
      log_gateway_requests: false
    use_boltdb_shipper_as_backup: false
    ingestername: loki-instance-0
    mode: RW
    ingesterdbretainperiod: 20m0s
    build_per_tenant_index: false
  tsdb_shipper:
    active_index_directory: ""
    shared_store: ""
    shared_store_key_prefix: index/
    cache_location: ""
    cache_ttl: 24h0m0s
    resync_interval: 5m0s
    query_ready_num_days: 0
    index_gateway_client:
      grpc_client_config:
        max_recv_msg_size: 104857600
        max_send_msg_size: 104857600
        grpc_compression: ""
        rate_limit: 0
        rate_limit_burst: 0
        backoff_on_ratelimits: false
        backoff_config:
          min_period: 100ms
          max_period: 10s
          max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
      log_gateway_requests: false
    use_boltdb_shipper_as_backup: false
    ingestername: loki-instance-0
    mode: RW
    ingesterdbretainperiod: 20m0s
chunk_store_config:
  chunk_cache_config:
    enable_fifocache: true
    default_validity: 1h0m0s
    background:
      writeback_goroutines: 10
      writeback_buffer: 10000
    memcached:
      expiration: 0s
      batch_size: 1024
      parallelism: 100
    memcached_client:
      host: ""
      service: memcached
      addresses: ""
      timeout: 100ms
      max_idle_conns: 16
      max_item_size: 0
      update_interval: 1m0s
      consistent_hash: true
      circuit_breaker_consecutive_failures: 10
      circuit_breaker_timeout: 10s
      circuit_breaker_interval: 10s
    redis:
      endpoint: ""
      master_name: ""
      timeout: 500ms
      expiration: 0s
      db: 0
      pool_size: 0
      username: ""
      password: ""
      tls_enabled: false
      tls_insecure_skip_verify: false
      idle_timeout: 0s
      max_connection_age: 0s
    embedded_cache:
      max_size_mb: 100
      ttl: 1h0m0s
    fifocache:
      max_size_bytes: 1GB
      max_size_items: 0
      ttl: 1h0m0s
      validity: 0s
      size: 0
      purgeinterval: 0s
    prefix: store.chunks-cache.
    async_cache_write_back_concurrency: 16
    async_cache_write_back_buffer_size: 500
  write_dedupe_cache_config:
    enable_fifocache: false
    default_validity: 1h0m0s
    background:
      writeback_goroutines: 10
      writeback_buffer: 10000
    memcached:
      expiration: 0s
      batch_size: 1024
      parallelism: 100
    memcached_client:
      host: ""
      service: memcached
      addresses: ""
      timeout: 100ms
      max_idle_conns: 16
      max_item_size: 0
      update_interval: 1m0s
      consistent_hash: true
      circuit_breaker_consecutive_failures: 10
      circuit_breaker_timeout: 10s
      circuit_breaker_interval: 10s
    redis:
      endpoint: ""
      master_name: ""
      timeout: 500ms
      expiration: 0s
      db: 0
      pool_size: 0
      username: ""
      password: ""
      tls_enabled: false
      tls_insecure_skip_verify: false
      idle_timeout: 0s
      max_connection_age: 0s
    embedded_cache:
      max_size_mb: 100
      ttl: 1h0m0s
    fifocache:
      max_size_bytes: 1GB
      max_size_items: 0
      ttl: 1h0m0s
      validity: 0s
      size: 0
      purgeinterval: 0s
    prefix: store.index-cache-write.
    async_cache_write_back_concurrency: 16
    async_cache_write_back_buffer_size: 500
  cache_lookups_older_than: 0s
  max_look_back_period: 31d
schema_config:
  configs:
  - from: "2020-09-07"
    store: boltdb-shipper
    object_store: s3
    schema: v11
    index:
      prefix: loki_index_
      period: 1d
      tags: {}
    chunks:
      prefix: ""
      period: 0s
      tags: {}
    row_shards: 16
compactor:
  working_directory: /loki/compactor
  shared_store: s3
  shared_store_key_prefix: index/
  compaction_interval: 5m0s
  apply_retention_interval: 0s
  retention_enabled: true
  retention_delete_delay: 2h0m0s
  retention_delete_worker_count: 150
  retention_table_timeout: 0s
  delete_batch_size: 70
  delete_request_cancel_period: 24h0m0s
  delete_max_interval: 0s
  max_compaction_parallelism: 1
  upload_parallelism: 10
  compactor_ring:
    kvstore:
      store: memberlist
      prefix: collectors/
      consul:
        host: localhost:8500
        acl_token: ""
        http_client_timeout: 20s
        consistent_reads: false
        watch_rate_limit: 1
        watch_burst_size: 1
        cas_retry_delay: 1s
      etcd:
        endpoints: []
        dial_timeout: 10s
        max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
        username: ""
        password: ""
      multi:
        primary: ""
        secondary: ""
        mirror_enabled: false
        mirror_timeout: 2s
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    tokens_file_path: ""
    zone_awareness_enabled: false
    instance_id: loki-instance-0
    instance_interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    instance_port: 0
    instance_addr: 127.0.0.1
    instance_availability_zone: ""
  _: false
  tables_to_compact: 0
  skip_latest_n_tables: 0
  deletion_mode: ""
compactor_grpc_client:
  max_recv_msg_size: 104857600
  max_send_msg_size: 104857600
  grpc_compression: ""
  rate_limit: 0
  rate_limit_burst: 0
  backoff_on_ratelimits: false
  backoff_config:
    min_period: 100ms
    max_period: 10s
    max_retries: 10
  tls_enabled: false
  tls_cert_path: ""
  tls_key_path: ""
  tls_ca_path: ""
  tls_server_name: ""
  tls_insecure_skip_verify: false
  tls_cipher_suites: ""
  tls_min_version: ""
limits_config:
  ingestion_rate_strategy: global
  ingestion_rate_mb: 4
  ingestion_burst_size_mb: 6
  max_label_name_length: 1024
  max_label_value_length: 2048
  max_label_names_per_series: 30
  reject_old_samples: true
  reject_old_samples_max_age: 1w
  creation_grace_period: 10m
  enforce_metric_name: false
  max_line_size: 0
  max_line_size_truncate: false
  increment_duplicate_timestamp: false
  max_streams_per_user: 0
  max_global_streams_per_user: 5000
  unordered_writes: true
  per_stream_rate_limit: 3145728
  per_stream_rate_limit_burst: 15728640
  max_chunks_per_query: 2000000
  max_query_series: 500
  max_query_lookback: 31d
  max_query_length: 30d1h
  max_query_parallelism: 32
  tsdb_max_query_parallelism: 512
  cardinality_limit: 100000
  max_streams_matchers_per_query: 1000
  max_concurrent_tail_requests: 10
  max_entries_limit_per_query: 5000
  max_cache_freshness_per_query: 10m
  max_queriers_per_tenant: 0
  query_ready_index_num_days: 0
  query_timeout: 5m
  split_queries_by_interval: 0s
  min_sharding_lookback: 0s
  ruler_evaluation_delay_duration: 0s
  ruler_max_rules_per_rule_group: 0
  ruler_max_rule_groups_per_tenant: 0
  ruler_alertmanager_config: null
  ruler_tenant_shard_size: 0
  ruler_remote_write_disabled: false
  ruler_remote_write_url: ""
  ruler_remote_write_timeout: 0s
  ruler_remote_write_headers: {}
  ruler_remote_write_queue_capacity: 0
  ruler_remote_write_queue_min_shards: 0
  ruler_remote_write_queue_max_shards: 0
  ruler_remote_write_queue_max_samples_per_send: 0
  ruler_remote_write_queue_batch_send_deadline: 0s
  ruler_remote_write_queue_min_backoff: 0s
  ruler_remote_write_queue_max_backoff: 0s
  ruler_remote_write_queue_retry_on_ratelimit: false
  ruler_remote_write_sigv4_config: null
  deletion_mode: filter-and-delete
  retention_period: 31d
  per_tenant_override_config: ""
  per_tenant_override_period: 10s
  allow_deletes: false
  shard_streams:
    enabled: false
    logging_enabled: false
    desired_rate: 3145728
frontend_worker:
  frontend_address: ""
  scheduler_address: ""
  dns_lookup_duration: 3s
  parallelism: 10
  match_max_concurrent: true
  id: ""
  grpc_client_config:
    max_recv_msg_size: 104857600
    max_send_msg_size: 104857600
    grpc_compression: ""
    rate_limit: 0
    rate_limit_burst: 0
    backoff_on_ratelimits: false
    backoff_config:
      min_period: 100ms
      max_period: 10s
      max_retries: 10
    tls_enabled: false
    tls_cert_path: ""
    tls_key_path: ""
    tls_ca_path: ""
    tls_server_name: ""
    tls_insecure_skip_verify: false
    tls_cipher_suites: ""
    tls_min_version: ""
table_manager:
  throughput_updates_disabled: false
  retention_deletes_enabled: true
  retention_period: 31d
  poll_interval: 2m0s
  creation_grace_period: 10m0s
  index_tables_provisioning:
    enable_ondemand_throughput_mode: false
    provisioned_write_throughput: 1000
    provisioned_read_throughput: 300
    write_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    read_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    enable_inactive_throughput_on_demand_mode: false
    inactive_write_throughput: 1
    inactive_read_throughput: 300
    inactive_write_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    inactive_read_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    inactive_write_scale_lastn: 4
    inactive_read_scale_lastn: 4
  chunk_tables_provisioning:
    enable_ondemand_throughput_mode: false
    provisioned_write_throughput: 1000
    provisioned_read_throughput: 300
    write_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    read_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    enable_inactive_throughput_on_demand_mode: false
    inactive_write_throughput: 1
    inactive_read_throughput: 300
    inactive_write_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    inactive_read_scale:
      enabled: false
      role_arn: ""
      min_capacity: 3000
      max_capacity: 6000
      out_cooldown: 1800
      in_cooldown: 1800
      target: 80
    inactive_write_scale_lastn: 4
    inactive_read_scale_lastn: 4
memberlist:
  node_name: loki-instance-0
  randomize_node_name: true
  stream_timeout: 10s
  retransmit_factor: 4
  pull_push_interval: 30s
  gossip_interval: 200ms
  gossip_nodes: 3
  gossip_to_dead_nodes_time: 30s
  dead_node_reclaim_time: 0s
  compression_enabled: true
  advertise_addr: ""
  advertise_port: 7946
  cluster_label: ""
  cluster_label_verification_disabled: false
  join_members:
  - loki-instance-0
  - loki-instance-1
  min_join_backoff: 1s
  max_join_backoff: 1m0s
  max_join_retries: 10
  abort_if_cluster_join_fails: false
  rejoin_interval: 0s
  left_ingesters_timeout: 5m0s
  leave_timeout: 20s
  message_history_buffer_bytes: 0
  bind_addr: []
  bind_port: 7946
  packet_dial_timeout: 2s
  packet_write_timeout: 5s
  tls_enabled: false
  tls_cert_path: ""
  tls_key_path: ""
  tls_ca_path: ""
  tls_server_name: ""
  tls_insecure_skip_verify: false
  tls_cipher_suites: ""
  tls_min_version: ""
runtime_config:
  period: 10s
  file: ""
tracing:
  enabled: true
analytics:
  reporting_enabled: true
legacy_read_target: true
common:
  path_prefix: /loki
  storage:
    s3:
      s3: ""
      s3forcepathstyle: false
      bucketnames: ""
      endpoint: ""
      region: ""
      access_key_id: ""
      secret_access_key: ""
      session_token: ""
      insecure: false
      sse_encryption: false
      http_config:
        idle_conn_timeout: 1m30s
        response_header_timeout: 0s
        insecure_skip_verify: false
        ca_file: ""
      signature_version: v4
      storage_class: STANDARD
      sse:
        type: ""
        kms_key_id: ""
        kms_encryption_context: ""
      backoff_config:
        min_period: 100ms
        max_period: 3s
        max_retries: 5
    gcs:
      bucket_name: ""
      service_account: ""
      chunk_buffer_size: 0
      request_timeout: 0s
      enable_opencensus: true
      enable_http2: true
    azure:
      environment: AzureGlobal
      account_name: ""
      account_key: ""
      container_name: loki
      endpoint_suffix: ""
      use_managed_identity: false
      use_federated_token: false
      user_assigned_id: ""
      use_service_principal: false
      client_id: ""
      client_secret: ""
      tenant_id: ""
      chunk_delimiter: '-'
      download_buffer_size: 512000
      upload_buffer_size: 256000
      upload_buffer_count: 1
      request_timeout: 30s
      max_retries: 5
      min_retry_delay: 10ms
      max_retry_delay: 500ms
    alibabacloud:
      bucket: ""
      endpoint: ""
      access_key_id: ""
      secret_access_key: ""
    bos:
      bucket_name: ""
      endpoint: bj.bcebos.com
      access_key_id: ""
      secret_access_key: ""
    swift:
      auth_version: 0
      auth_url: ""
      internal: false
      username: ""
      user_domain_name: ""
      user_domain_id: ""
      user_id: ""
      password: ""
      domain_id: ""
      domain_name: ""
      project_id: ""
      project_name: ""
      project_domain_id: ""
      project_domain_name: ""
      region_name: ""
      container_name: ""
      max_retries: 3
      connect_timeout: 10s
      request_timeout: 5s
    filesystem:
      chunks_directory: ""
      rules_directory: ""
    hedging:
      at: 0s
      up_to: 2
      max_per_second: 5
  persist_tokens: false
  replication_factor: 1
  ring:
    kvstore:
      store: memberlist
      prefix: collectors/
      consul:
        host: localhost:8500
        acl_token: ""
        http_client_timeout: 20s
        consistent_reads: false
        watch_rate_limit: 1
        watch_burst_size: 1
        cas_retry_delay: 1s
      etcd:
        endpoints: []
        dial_timeout: 10s
        max_retries: 10
        tls_enabled: false
        tls_cert_path: ""
        tls_key_path: ""
        tls_ca_path: ""
        tls_server_name: ""
        tls_insecure_skip_verify: false
        tls_cipher_suites: ""
        tls_min_version: ""
        username: ""
        password: ""
      multi:
        primary: ""
        secondary: ""
        mirror_enabled: false
        mirror_timeout: 2s
    heartbeat_period: 15s
    heartbeat_timeout: 1m0s
    tokens_file_path: ""
    zone_awareness_enabled: false
    instance_id: loki-instance-0
    instance_interface_names:
    - ens2
    - ens4
    - docker0
    - br-a369ef4d6f79
    - lo
    instance_port: 0
    instance_addr: 127.0.0.1
    instance_availability_zone: ""
  instance_interface_names:
  - ens2
  - ens4
  - docker0
  - br-a369ef4d6f79
  instance_addr: ""
  compactor_address: ""
  compactor_grpc_address: ""
shutdown_delay: 0s

Sounds like the two instances don’t see each other as part of the ring membership. What do you see from /ring?

Hi and thx for your help.

Yes my ring seem be ok:

All services are running:

image

Couple of things to potentially try:

  1. I notice you have max_chunk_age: 2h0m0s, if you try to query logs older than 2 hours (or 3 hours), do you notice the same split behavior?

  2. You have max_chunk_age: 2h0m0s and query_ingesters_within: 3h0m0s, I would change query_ingesters_within to match 2h as well.

  3. Your query frontend isn’t actually configured. You need to set one of two configurations, see Query Frontend | Grafana Loki documentation.

Hi @tonyswumac

You are right !

If I query logs older than 2 hours, any split are make.

For you, what is the good way ? Set max_chunk_age=0s ?

For --frontend.downstream-url, i need to set here the public gateway on my Loki cluster ?

For example, https://loki.example.com ?

Many thx :slight_smile:

  1. If you are not seeing split behavior after 2 hours, then your problem is most certainly ring related. I went back and looked at your screenshot again, and I noticed that both of your instance’s listed address is 127.0.0.1:9095, that could be your problem. Each instance should have an unique IP that they can communicate to each other with. In your case neither instance can communicate with the other, because when they try to hit 127.0.0.1:9095 they are hitting themselves.

  2. According to documentation, pull method is preferred, so you should try and set frontend_address, since you are using monolithic mode it should be some sort of service discovery record that can point to either instance on the gRPC port. I’d also recommend you to focus on fixing #1 first, since I don’t think this is related to your problem judging from your reply. One problem at a time.

HI,

Some logs in debug:

level=warn ts=2023-07-19T09:31:53.578102242Z caller=store.go:51 msg="running with DEPRECATED flag -store.max-look-back-period, use -querier.max-query-lookback instead."
level=warn ts=2023-07-19T09:31:53.578247223Z caller=loki.go:286 msg="per-tenant timeout not configured, using default engine timeout (\"5m0s\"). This behavior will change in the next major to always use the default per-tenant timeout (\"5m\")."
level=info ts=2023-07-19T09:31:53.580552559Z caller=main.go:108 msg="Starting Loki" version="(version=2.8.2, branch=HEAD, revision=9f809eda7)"
level=info ts=2023-07-19T09:31:53.581206554Z caller=server.go:323 http=[::]:3100 grpc=[::]:9095 msg="server listening on addresses"
level=info ts=2023-07-19T09:31:53.581474036Z caller=modules.go:894 msg="Ruler storage is not configured; ruler will not be started."
level=info ts=2023-07-19T09:31:53.582266781Z caller=memberlist_client.go:437 msg="Using memberlist cluster label and node name" cluster_label= node=loki-instance-1-6324eab7
ts=2023-07-19T09:31:53.582333692Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=warn ts=2023-07-19T09:31:53.583326939Z caller=cache.go:114 msg="fifocache config is deprecated. use embedded-cache instead"
level=warn ts=2023-07-19T09:31:53.583346329Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - chunksembedded-cache"
level=info ts=2023-07-19T09:31:53.583965353Z caller=table_manager.go:134 msg="uploading tables"
level=info ts=2023-07-19T09:31:53.584787159Z caller=table_manager.go:404 msg="loading local table loki_index_19554"
level=debug ts=2023-07-19T09:31:53.58491801Z caller=table.go:98 table-name=loki_index_19554 msg="opening locally present files for table loki_index_19554" files=[0xc000051e00]
ts=2023-07-19T09:31:53.58496319Z caller=spanlogger.go:85 table-name=loki_index_19554 user-id=fake level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.58498404Z caller=index_set.go:293 table-name=loki_index_19554 user-id=fake msg="syncing files for table loki_index_19554"
ts=2023-07-19T09:31:53.58502857Z caller=spanlogger.go:85 level=info msg="building index list cache"
level=debug ts=2023-07-19T09:31:53.590116565Z caller=tcp_transport.go:393 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=192.168.1.101 advertisePort=7946
level=debug ts=2023-07-19T09:31:53.597615028Z caller=tcp_transport.go:393 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=192.168.1.101 advertisePort=7946
level=info ts=2023-07-19T09:31:53.597823399Z caller=memberlist_client.go:543 msg="memberlist fast-join starting" nodes_found=2 to_join=4
ts=2023-07-19T09:31:53.599188129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  192.168.1.101:7946"
ts=2023-07-19T09:31:53.599252789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.1.101:53184"
ts=2023-07-19T09:31:53.601473674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  192.168.1.102:7946"
level=info ts=2023-07-19T09:31:53.603796281Z caller=memberlist_client.go:563 msg="memberlist fast-join finished" joined_nodes=2 elapsed_time=5.978092ms
level=info ts=2023-07-19T09:31:53.603873571Z caller=memberlist_client.go:576 msg="joining memberlist cluster" join_members=loki-instance-0.priv,loki-instance-1.priv
ts=2023-07-19T09:31:53.605820845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  192.168.1.102:7946"
ts=2023-07-19T09:31:53.608908846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  192.168.1.101:7946"
ts=2023-07-19T09:31:53.608964206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.1.101:53190"
level=info ts=2023-07-19T09:31:53.609912883Z caller=memberlist_client.go:595 msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=6.040802ms
ts=2023-07-19T09:31:53.673023632Z caller=spanlogger.go:85 level=info msg="index list cache built" duration=87.987772ms
level=debug ts=2023-07-19T09:31:53.673072083Z caller=index_set.go:300 table-name=loki_index_19554 user-id=fake msg="updates for table loki_index_19554. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673101443Z caller=spanlogger.go:85 table-name=loki_index_19554 user-id=fake level=debug msg="finished syncing files"
ts=2023-07-19T09:31:53.673194023Z caller=spanlogger.go:85 table-name=loki_index_19554 level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673199683Z caller=index_set.go:293 table-name=loki_index_19554 msg="syncing files for table loki_index_19554"
level=debug ts=2023-07-19T09:31:53.673209603Z caller=index_set.go:300 table-name=loki_index_19554 msg="updates for table loki_index_19554. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673215884Z caller=spanlogger.go:85 table-name=loki_index_19554 level=debug msg="finished syncing files"
level=info ts=2023-07-19T09:31:53.673234794Z caller=table_manager.go:404 msg="loading local table loki_index_19555"
level=debug ts=2023-07-19T09:31:53.673264714Z caller=table.go:98 table-name=loki_index_19555 msg="opening locally present files for table loki_index_19555" files=[0xc00071ae00]
ts=2023-07-19T09:31:53.673286254Z caller=spanlogger.go:85 table-name=loki_index_19555 user-id=fake level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673290904Z caller=index_set.go:293 table-name=loki_index_19555 user-id=fake msg="syncing files for table loki_index_19555"
level=debug ts=2023-07-19T09:31:53.673301394Z caller=index_set.go:300 table-name=loki_index_19555 user-id=fake msg="updates for table loki_index_19555. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673307414Z caller=spanlogger.go:85 table-name=loki_index_19555 user-id=fake level=debug msg="finished syncing files"
ts=2023-07-19T09:31:53.673325054Z caller=spanlogger.go:85 table-name=loki_index_19555 level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673330604Z caller=index_set.go:293 table-name=loki_index_19555 msg="syncing files for table loki_index_19555"
level=debug ts=2023-07-19T09:31:53.673337024Z caller=index_set.go:300 table-name=loki_index_19555 msg="updates for table loki_index_19555. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673341884Z caller=spanlogger.go:85 table-name=loki_index_19555 level=debug msg="finished syncing files"
level=info ts=2023-07-19T09:31:53.673349534Z caller=table_manager.go:404 msg="loading local table loki_index_19556"
level=debug ts=2023-07-19T09:31:53.673377375Z caller=table.go:98 table-name=loki_index_19556 msg="opening locally present files for table loki_index_19556" files=[0xc00071b040]
ts=2023-07-19T09:31:53.673427435Z caller=spanlogger.go:85 table-name=loki_index_19556 user-id=fake level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673436495Z caller=index_set.go:293 table-name=loki_index_19556 user-id=fake msg="syncing files for table loki_index_19556"
level=debug ts=2023-07-19T09:31:53.673453895Z caller=index_set.go:300 table-name=loki_index_19556 user-id=fake msg="updates for table loki_index_19556. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673461065Z caller=spanlogger.go:85 table-name=loki_index_19556 user-id=fake level=debug msg="finished syncing files"
ts=2023-07-19T09:31:53.673501775Z caller=spanlogger.go:85 table-name=loki_index_19556 level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673513126Z caller=index_set.go:293 table-name=loki_index_19556 msg="syncing files for table loki_index_19556"
level=debug ts=2023-07-19T09:31:53.673520816Z caller=index_set.go:300 table-name=loki_index_19556 msg="updates for table loki_index_19556. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673526646Z caller=spanlogger.go:85 table-name=loki_index_19556 level=debug msg="finished syncing files"
level=info ts=2023-07-19T09:31:53.673535766Z caller=table_manager.go:404 msg="loading local table loki_index_19557"
level=debug ts=2023-07-19T09:31:53.673588716Z caller=table.go:98 table-name=loki_index_19557 msg="opening locally present files for table loki_index_19557" files="[0xc00071b2c0 0xc00071b280]"
ts=2023-07-19T09:31:53.673622256Z caller=spanlogger.go:85 table-name=loki_index_19557 user-id=fake level=debug msg="opened 0 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673628106Z caller=index_set.go:293 table-name=loki_index_19557 user-id=fake msg="syncing files for table loki_index_19557"
level=debug ts=2023-07-19T09:31:53.673636326Z caller=index_set.go:300 table-name=loki_index_19557 user-id=fake msg="updates for table loki_index_19557. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673642776Z caller=spanlogger.go:85 table-name=loki_index_19557 user-id=fake level=debug msg="finished syncing files"
ts=2023-07-19T09:31:53.673839728Z caller=spanlogger.go:85 table-name=loki_index_19557 level=debug msg="opened 1 local files, now starting sync operation"
level=debug ts=2023-07-19T09:31:53.673906718Z caller=index_set.go:293 table-name=loki_index_19557 msg="syncing files for table loki_index_19557"
level=debug ts=2023-07-19T09:31:53.673917378Z caller=index_set.go:300 table-name=loki_index_19557 msg="updates for table loki_index_19557. toDownload: [], toDelete: []"
ts=2023-07-19T09:31:53.673923948Z caller=spanlogger.go:85 table-name=loki_index_19557 level=debug msg="finished syncing files"
level=info ts=2023-07-19T09:31:53.673932698Z caller=table_manager.go:262 msg="query readiness setup completed" duration=1.96µs distinct_users_len=0
level=info ts=2023-07-19T09:31:53.673962419Z caller=shipper.go:131 msg="starting index shipper in RW mode"
level=info ts=2023-07-19T09:31:53.67412077Z caller=shipper_index_client.go:78 msg="starting boltdb shipper in RW mode"
level=info ts=2023-07-19T09:31:53.674245421Z caller=table_manager.go:166 msg="handing over indexes to shipper"
level=info ts=2023-07-19T09:31:53.674794104Z caller=modules.go:913 msg="RulerStorage is nil.  Not starting the ruler."
level=debug ts=2023-07-19T09:31:53.67700234Z caller=worker_service.go:151 msg="determining if querier is running as standalone target" runningStandalone=false queryFrontendEnabled=false queryScheduleEnabled=false readEnabled=false allEnabled=true
level=info ts=2023-07-19T09:31:53.67706901Z caller=worker.go:112 msg="Starting querier worker using query-scheduler and scheduler ring for addresses"
level=debug ts=2023-07-19T09:31:53.678345819Z caller=modules.go:681 msg="initializing query frontend tripperware"
level=warn ts=2023-07-19T09:31:53.67845724Z caller=experimental.go:20 msg="experimental feature in use" feature="In-memory (FIFO) cache - frontend.embedded-cache"
level=debug ts=2023-07-19T09:31:53.678778652Z caller=modules.go:754 msg="initializing query frontend" config="{Handler:{LogQueriesLongerThan:5s MaxBodySize:10485760 QueryStatsEnabled:false} FrontendV1:{MaxOutstandingPerTenant:4096 QuerierForgetDelay:0s} FrontendV2:{SchedulerAddress: DNSLookupPeriod:10s WorkerConcurrency:5 GRPCClientConfig:{MaxRecvMsgSize:104857600 MaxSendMsgSize:104857600 GRPCCompression: RateLimit:0 RateLimitBurst:0 BackoffOnRatelimits:false BackoffConfig:{MinBackoff:100ms MaxBackoff:10s MaxRetries:10} TLSEnabled:false TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion:}} GracefulShutdownTimeout:5m0s InfNames:[ens2 ens4 br-31bfea494ff4 docker0 lo] Addr: Port:0} CompressResponses:true DownstreamURL: TailProxyURL: TLS:{CertPath: KeyPath: CAPath: ServerName: InsecureSkipVerify:false CipherSuites: MinVersion:}}"
level=debug ts=2023-07-19T09:31:53.678950213Z caller=modules.go:780 msg="using query frontend" version=v2
level=info ts=2023-07-19T09:31:53.683106752Z caller=module_service.go:82 msg=initialising module=server
level=debug ts=2023-07-19T09:31:53.683474744Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.683565665Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.683533115Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.683600835Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=ring
level=debug ts=2023-07-19T09:31:53.683578695Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=usage-report
level=debug ts=2023-07-19T09:31:53.683621545Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ingester-querier
level=debug ts=2023-07-19T09:31:53.683620385Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=ring
level=debug ts=2023-07-19T09:31:53.683631326Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend-tripperware waiting_for=server
level=info ts=2023-07-19T09:31:53.683636516Z caller=module_service.go:82 msg=initialising module=cache-generation-loader
level=debug ts=2023-07-19T09:31:53.683641876Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=query-frontend-tripperware
level=debug ts=2023-07-19T09:31:53.683664096Z caller=module_service.go:72 msg="module waiting for initialization" module=usage-report waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.683681906Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.683693666Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2023-07-19T09:31:53.683705236Z caller=module_service.go:82 msg=initialising module=memberlist-kv
level=debug ts=2023-07-19T09:31:53.683852737Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=ring
level=debug ts=2023-07-19T09:31:53.683864127Z caller=module_service.go:72 msg="module waiting for initialization" module=usage-report waiting_for=ring
level=debug ts=2023-07-19T09:31:53.683870617Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2023-07-19T09:31:53.683876547Z caller=module_service.go:82 msg=initialising module=ring
level=info ts=2023-07-19T09:31:53.683917648Z caller=module_service.go:82 msg=initialising module=query-frontend-tripperware
level=debug ts=2023-07-19T09:31:53.683967448Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=query-scheduler
level=debug ts=2023-07-19T09:31:53.684095169Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=ring
level=debug ts=2023-07-19T09:31:53.684111559Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=server
level=debug ts=2023-07-19T09:31:53.684136869Z caller=module_service.go:72 msg="module waiting for initialization" module=query-scheduler waiting_for=usage-report
level=debug ts=2023-07-19T09:31:53.684144149Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2023-07-19T09:31:53.684148249Z caller=module_service.go:82 msg=initialising module=store
level=debug ts=2023-07-19T09:31:53.684158929Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=server
level=debug ts=2023-07-19T09:31:53.684156979Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=server
level=debug ts=2023-07-19T09:31:53.684162989Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester-querier waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.684166579Z caller=module_service.go:72 msg="module waiting for initialization" module=compactor waiting_for=usage-report
level=info ts=2023-07-19T09:31:53.684167899Z caller=module_service.go:82 msg=initialising module=ingester-querier
level=debug ts=2023-07-19T09:31:53.684172639Z caller=module_service.go:72 msg="module waiting for initialization" module=usage-report waiting_for=server
level=info ts=2023-07-19T09:31:53.684177939Z caller=module_service.go:82 msg=initialising module=usage-report
level=debug ts=2023-07-19T09:31:53.684176609Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=server
level=debug ts=2023-07-19T09:31:53.684188069Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=usage-report
level=debug ts=2023-07-19T09:31:53.684195579Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.684200199Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=query-scheduler
level=debug ts=2023-07-19T09:31:53.6842279Z caller=module_service.go:72 msg="module waiting for initialization" module=distributor waiting_for=memberlist-kv
level=info ts=2023-07-19T09:31:53.68423746Z caller=module_service.go:82 msg=initialising module=distributor
level=info ts=2023-07-19T09:31:53.68425586Z caller=module_service.go:82 msg=initialising module=compactor
level=info ts=2023-07-19T09:31:53.6842959Z caller=module_service.go:82 msg=initialising module=query-scheduler
level=debug ts=2023-07-19T09:31:53.684425061Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2023-07-19T09:31:53.684434051Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2023-07-19T09:31:53.684438211Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2023-07-19T09:31:53.684442151Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2023-07-19T09:31:53.684446031Z caller=module_service.go:82 msg=initialising module=ingester
level=info ts=2023-07-19T09:31:53.684463861Z caller=ingester.go:416 msg="recovering from checkpoint"
level=info ts=2023-07-19T09:31:53.684496622Z caller=basic_lifecycler.go:261 msg="instance not found in the ring" instance=loki-instance-1 ring=compactor
level=info ts=2023-07-19T09:31:53.684514002Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2023-07-19T09:31:53.684601212Z caller=compactor.go:332 msg="waiting until compactor is JOINING in the ring"
level=info ts=2023-07-19T09:31:53.684608572Z caller=compactor.go:336 msg="compactor is JOINING in the ring"
level=debug ts=2023-07-19T09:31:53.684621252Z caller=memberlist_client.go:883 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2023-07-19T09:31:53.684645473Z caller=lifecycler.go:547 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2023-07-19T09:31:53.684677563Z caller=lifecycler.go:576 msg="instance not found in ring, adding with no tokens" ring=distributor
level=debug ts=2023-07-19T09:31:53.684745513Z caller=lifecycler.go:412 msg="JoinAfter expired" ring=distributor
level=info ts=2023-07-19T09:31:53.684758473Z caller=lifecycler.go:416 msg="auto-joining cluster after timeout" ring=distributor
level=info ts=2023-07-19T09:31:53.684849964Z caller=basic_lifecycler.go:261 msg="instance not found in the ring" instance=loki-instance-1 ring=scheduler
level=info ts=2023-07-19T09:31:53.684866284Z caller=basic_lifecycler_delegates.go:63 msg="not loading tokens from file, tokens file path is empty"
level=debug ts=2023-07-19T09:31:53.684993325Z caller=memberlist_client.go:883 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2023-07-19T09:31:53.685054425Z caller=scheduler.go:616 msg="waiting until scheduler is JOINING in the ring"
level=info ts=2023-07-19T09:31:53.685063315Z caller=scheduler.go:620 msg="scheduler is JOINING in the ring"
level=debug ts=2023-07-19T09:31:53.685089716Z caller=memberlist_client.go:883 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2023-07-19T09:31:53.685830861Z caller=ingester.go:432 msg="recovered WAL checkpoint recovery finished" elapsed=1.37597ms errors=false
level=info ts=2023-07-19T09:31:53.685848951Z caller=ingester.go:438 msg="recovering from WAL"
level=info ts=2023-07-19T09:31:53.688691031Z caller=ingester.go:454 msg="WAL segment recovery finished" elapsed=4.233609ms errors=false
level=info ts=2023-07-19T09:31:53.688743701Z caller=ingester.go:402 msg="closing recoverer"
level=info ts=2023-07-19T09:31:53.688759191Z caller=ingester.go:410 msg="WAL recovery finished" time=4.30345ms
level=info ts=2023-07-19T09:31:53.689216874Z caller=lifecycler.go:547 msg="not loading tokens from file, tokens file path is empty"
level=info ts=2023-07-19T09:31:53.689313125Z caller=lifecycler.go:576 msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2023-07-19T09:31:53.689494296Z caller=lifecycler.go:412 msg="JoinAfter expired" ring=ingester
level=info ts=2023-07-19T09:31:53.689509696Z caller=lifecycler.go:416 msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2023-07-19T09:31:53.689684077Z caller=memberlist_client.go:883 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2023-07-19T09:31:53.689702397Z caller=wal.go:156 msg=started component=wal
level=info ts=2023-07-19T09:31:54.685315349Z caller=scheduler.go:630 msg="waiting until scheduler is ACTIVE in the ring"
level=info ts=2023-07-19T09:31:54.6854666Z caller=compactor.go:346 msg="waiting until compactor is ACTIVE in the ring"
level=info ts=2023-07-19T09:31:54.68551429Z caller=compactor.go:350 msg="compactor is ACTIVE in the ring"
level=info ts=2023-07-19T09:31:54.846409275Z caller=scheduler.go:634 msg="scheduler is ACTIVE in the ring"
level=debug ts=2023-07-19T09:31:54.846497446Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=ring
level=debug ts=2023-07-19T09:31:54.846510746Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=server
level=debug ts=2023-07-19T09:31:54.846517446Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=store
level=debug ts=2023-07-19T09:31:54.846522656Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=usage-report
level=debug ts=2023-07-19T09:31:54.846527326Z caller=module_service.go:72 msg="module waiting for initialization" module=querier waiting_for=cache-generation-loader
level=info ts=2023-07-19T09:31:54.846533036Z caller=module_service.go:82 msg=initialising module=querier
level=debug ts=2023-07-19T09:31:54.846556256Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=ring
level=debug ts=2023-07-19T09:31:54.846600526Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=server
level=debug ts=2023-07-19T09:31:54.846611956Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=usage-report
level=debug ts=2023-07-19T09:31:54.846622726Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=cache-generation-loader
level=debug ts=2023-07-19T09:31:54.846637017Z caller=module_service.go:72 msg="module waiting for initialization" module=query-frontend waiting_for=memberlist-kv
level=info ts=2023-07-19T09:31:54.846644747Z caller=module_service.go:82 msg=initialising module=query-frontend
level=info ts=2023-07-19T09:31:54.846763487Z caller=loki.go:499 msg="Loki started"
level=debug ts=2023-07-19T09:31:56.244454376Z caller=logging.go:76 traceID=5871c0316fb1cabd orgID=fake msg="GET /ready (503) 117.271µs"
level=debug ts=2023-07-19T09:31:56.748478546Z caller=logging.go:76 traceID=6f0fe1acc5162a3f orgID=fake msg="GET /ready (503) 122.511µs"
level=debug ts=2023-07-19T09:31:57.847525945Z caller=ring_watcher.go:88 component=querier-scheduler-worker msg="adding connection to address: 127.0.0.1:9095"
level=info ts=2023-07-19T09:31:57.847588795Z caller=worker.go:209 msg="adding connection" addr=127.0.0.1:9095
level=info ts=2023-07-19T09:31:57.847585295Z caller=scheduler.go:681 msg="this scheduler is in the ReplicationSet, will now accept requests."
level=debug ts=2023-07-19T09:31:57.847876067Z caller=ring_watcher.go:88 component=querier-scheduler-worker msg="adding connection to address: 127.0.0.1:9095"
level=debug ts=2023-07-19T09:31:57.849082906Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849097246Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849099596Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849149956Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849157776Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849177826Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849182076Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849193567Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849205997Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=debug ts=2023-07-19T09:31:57.849221357Z caller=scheduler.go:443 msg="querier connected" querier=loki-instance-1
level=info ts=2023-07-19T09:31:59.685952654Z caller=compactor.go:411 msg="this instance has been chosen to run the compactor, starting compactor"
level=info ts=2023-07-19T09:31:59.686106125Z caller=compactor.go:440 msg="waiting 5m0s for ring to stay stable and previous compactions to finish before starting compactor"
level=debug ts=2023-07-19T09:32:01.751975586Z caller=logging.go:76 traceID=040110bb26246518 orgID=fake msg="GET /ready (503) 119.671µs"
level=debug ts=2023-07-19T09:32:04.847322931Z caller=ring_watcher.go:88 component=frontend-scheduler-worker msg="adding connection to address: 127.0.0.1:9095"
level=info ts=2023-07-19T09:32:04.847363371Z caller=frontend_scheduler_worker.go:107 msg="adding connection to scheduler" addr=127.0.0.1:9095
level=debug ts=2023-07-19T09:32:04.847897665Z caller=ring_watcher.go:88 component=frontend-scheduler-worker msg="adding connection to address: 127.0.0.1:9095"
level=debug ts=2023-07-19T09:32:04.848494579Z caller=scheduler.go:337 msg="frontend connected" address=10.197.210.89:9095
level=debug ts=2023-07-19T09:32:04.848501649Z caller=scheduler.go:337 msg="frontend connected" address=10.197.210.89:9095
level=debug ts=2023-07-19T09:32:04.848503959Z caller=scheduler.go:337 msg="frontend connected" address=10.197.210.89:9095
level=debug ts=2023-07-19T09:32:04.84863082Z caller=scheduler.go:337 msg="frontend connected" address=10.197.210.89:9095
level=debug ts=2023-07-19T09:32:04.84865342Z caller=scheduler.go:337 msg="frontend connected" address=10.197.210.89:9095
ts=2023-07-19T09:32:05.023784272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.1.102:36300"
level=debug ts=2023-07-19T09:32:06.755293127Z caller=logging.go:76 traceID=59594a8e90bc23b8 orgID=fake msg="GET /ready (503) 103.211µs"
level=debug ts=2023-07-19T09:32:11.75890456Z caller=logging.go:76 traceID=4707da0deda72e8d orgID=fake msg="GET /ready (200) 130.291µs"
level=debug ts=2023-07-19T09:32:12.26278989Z caller=logging.go:76 traceID=7030013e65ec3f4f orgID=fake msg="GET /ready (200) 75.05µs"
level=debug ts=2023-07-19T09:32:17.266336334Z caller=logging.go:76 traceID=5b27c4b9020c263f orgID=fake msg="GET /ready (200) 76.05µs"
level=debug ts=2023-07-19T09:32:22.273005028Z caller=logging.go:76 traceID=56c0620e9097bd7d orgID=fake msg="GET /ready (200) 63.92µs"
level=debug ts=2023-07-19T09:32:22.426940282Z caller=push.go:150 org_id=fake msg="push request parsed" path=/loki/api/v1/push contentType=application/x-protobuf contentEncoding= bodySize="3.1 kB" streams=1 entries=16 streamLabelsSize="274 B" entriesSize="7.7 kB" totalSize="7.9 kB" mostRecentLagMs=1252
level=debug ts=2023-07-19T09:32:22.428585534Z caller=grpc_logging.go:46 method=/logproto.Pusher/Push duration=158.881µs msg="gRPC (success)"
level=debug ts=2023-07-19T09:32:22.428858746Z caller=logging.go:76 traceID=23d042429fcb1bb9 orgID=fake msg="POST /loki/api/v1/push (204) 2.170805ms"
level=debug ts=2023-07-19T09:32:23.685431353Z caller=grpc_logging.go:46 method=/grpc.health.v1.Health/Check duration=31.551µs msg="gRPC (success)"
level=debug ts=2023-07-19T09:32:27.276155035Z caller=logging.go:76 traceID=1cb4408159b3ba8a orgID=fake msg="GET /ready (200) 75.061µs"
ts=2023-07-19T09:32:31.35645741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-instance-0-ab7e7b10 192.168.1.102:7946"
level=debug ts=2023-07-19T09:32:32.27950432Z caller=logging.go:76 traceID=2ca4bc8892348c10 orgID=fake msg="GET /ready (200) 73.65µs"
ts=2023-07-19T09:32:35.027274676Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=192.168.1.102:41912"
level=debug ts=2023-07-19T09:32:37.283640384Z caller=logging.go:76 traceID=01120e290799efb9 orgID=fake msg="GET /ready (200) 72.521µs"
level=debug ts=2023-07-19T09:32:38.426258481Z caller=push.go:150 org_id=fake msg="push request parsed" path=/loki/api/v1/push contentType=application/x-protobuf contentEncoding= bodySize="3.0 kB" streams=1 entries=16 streamLabelsSize="274 B" entriesSize="7.7 kB" totalSize="7.9 kB" mostRecentLagMs=1142
level=debug ts=2023-07-19T09:32:38.426813145Z caller=grpc_logging.go:46 method=/logproto.Pusher/Push duration=127.571µs msg="gRPC (success)"
level=debug ts=2023-07-19T09:32:38.427017757Z caller=logging.go:76 traceID=033390d617082db8 orgID=fake msg="POST /loki/api/v1/push (204) 989.157µs"
level=debug ts=2023-07-19T09:32:38.685919665Z caller=grpc_logging.go:46 method=/grpc.health.v1.Health/Check duration=99.21µs msg="gRPC (success)"
level=debug ts=2023-07-19T09:32:42.288467574Z caller=logging.go:76 traceID=67871fed4f9da9b2 orgID=fake msg="GET /ready (200) 78.12µs"
level=debug ts=2023-07-19T09:32:47.29185898Z caller=logging.go:76 traceID=68dcff6945fe5822 orgID=fake msg="GET /ready (200) 67.551µs"

With the following config:

auth_enabled: false

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  retention_period: 744h
  split_queries_by_interval: 0s
  max_cache_freshness_per_query: 10m
  max_query_parallelism: 32

http_prefix:

server:
  http_listen_port: 3100
  log_level: debug

compactor:
  working_directory: /loki/compactor
  retention_enabled: true
  retention_delete_delay: 2h
  shared_store: s3
  compaction_interval: 5m
  compactor_ring:
    kvstore:
        store: memberlist

common:
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: memberlist
  replication_factor: 1
  path_prefix: /loki

memberlist:
  node_name: loki-instance-1
  join_members: 
        - "loki-instance-0.priv"
        - "loki-instance-1.priv"
    
schema_config:
  configs:
  - from: 2020-09-07
    store: boltdb-shipper
    object_store: s3
    schema: v11
    index:
      prefix: loki_index_
      period: 24h

storage_config:
  boltdb_shipper:
    shared_store: s3
    active_index_directory: /loki/index
    cache_location: /loki/boltdb-cache
    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
  aws:
    s3: "s3://[...]:[...]@s3.fr-par.scw.cloud/[...]"
    region: fr-par
    s3forcepathstyle: true

chunk_store_config:
  max_look_back_period: 744h

table_manager:
  retention_deletes_enabled: true
  retention_period: 744h

querier:
  query_ingesters_within: 2h

query_scheduler:
  max_outstanding_requests_per_tenant: 4096
  scheduler_ring:
    kvstore:
      store: memberlist

frontend:
  log_queries_longer_than: 5s
  max_outstanding_per_tenant: 4096
  compress_responses: true

query_range:
  align_queries_with_step: true
  max_retries: 5
  parallelise_shardable_queries: false
  cache_results: true
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        ttl: 24h

distributor:
  ring:
    kvstore:
      store: memberlist

ingester:
  max_chunk_age: 2h
  lifecycler:
    ring:
      kvstore:
        store: memberlist
      replication_factor: 1
  chunk_idle_period: 30m
  chunk_block_size: 262144
  chunk_encoding: snappy
  chunk_retain_period: 1m
  max_transfer_retries: 0
  wal:
    dir: /loki/wal

index_gateway:
  ring:
    kvstore:
      store: memberlist

Couple of things to try:

  1. Remove the instance_addr configuration:
common:
  ring:
    # instance_addr: 127.0.0.1
    kvstore:
      store: memberlist
  replication_factor: 1
  path_prefix: /loki
  1. Remove the node_name from memberlist:
memberlist:
  # node_name: loki-instance-1
  join_members: 
        - "loki-instance-0.priv"
        - "loki-instance-1.priv"

See if that helps.

This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.