Setup Loki Distributed using the official helm charts.
https://github.com/grafana/helm-charts/tree/main/charts/loki-distributed
Values.yaml file
nameOverride: null
# -- Overrides the chart's computed full name
fullnameOverride: null
# -- Image pull secrets for Docker images
imagePullSecrets: []
loki:
# Configures the readiness probe for all of the Loki pods
readinessProbe:
httpGet:
path: /ready
port: http
initialDelaySeconds: 30
timeoutSeconds: 1
image:
# -- The Docker registry
registry: docker.io
# -- Docker image repository
repository: grafana/loki
# -- Overrides the image tag whose default is the chart's appVersion
tag: null
# -- Docker image pull policy
pullPolicy: IfNotPresent
# -- Common annotations for all pods
podAnnotations: {}
# -- The number of old ReplicaSets to retain to allow rollback
revisionHistoryLimit: 10
# -- The SecurityContext for Loki pods
podSecurityContext:
fsGroup: 10001
runAsGroup: 10001
runAsNonRoot: true
runAsUser: 10001
# -- The SecurityContext for Loki containers
containerSecurityContext:
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
allowPrivilegeEscalation: false
# -- Specify an existing secret containing loki configuration. If non-empty, overrides `loki.config`
existingSecretForConfig: ""
# -- Config file contents for Loki
# @default -- See values.yaml
config: |
auth_enabled: false
server:
log_level: info
http_listen_port: 3100
distributor:
ring:
kvstore:
store: memberlist
ingester:
max_transfer_retries: 0
chunk_idle_period: 1h
chunk_target_size: 1536000
max_chunk_age: 1h
lifecycler:
join_after: 0s
ring:
kvstore:
store: memberlist
memberlist:
join_members:
- {{ include "loki.fullname" . }}-memberlist
limits_config:
ingestion_rate_mb: 10
ingestion_burst_size_mb: 20
max_concurrent_tail_requests: 20
max_cache_freshness_per_query: 10m
schema_config:
configs:
- from: 2020-09-07
store: boltdb-shipper
object_store: aws
schema: v11
index:
prefix: loki_index_
period: 24h
storage_config:
aws:
s3: s3://eu-west-1
bucketnames: loki-dev-v1
boltdb_shipper:
active_index_directory: /var/loki/index
shared_store: s3
cache_location: /var/loki/cache
index_gateway_client:
server_address: dns:///{{ include "loki.indexGatewayFullname" . }}:9095
query_range:
align_queries_with_step: true
max_retries: 5
split_queries_by_interval: 15m
cache_results: true
results_cache:
cache:
enable_fifocache: true
fifocache:
max_size_items: 1024
validity: 24h
frontend_worker:
frontend_address: {{ include "loki.queryFrontendFullname" . }}:9095
frontend:
log_queries_longer_than: 5s
compress_responses: true
tail_proxy_url: http://{{ include "loki.querierFullname" . }}:3100
serviceAccount:
# -- Specifies whether a ServiceAccount should be created
create: true
# -- The name of the ServiceAccount to use.
# If not set and create is true, a name is generated using the fullname template
name: null
# -- Image pull secrets for the service account
imagePullSecrets: []
# -- Annotations for the service account
annotations: {}
# -- Set this toggle to false to opt out of automounting API credentials for the service account
automountServiceAccountToken: true
# RBAC configuration
rbac:
# -- If enabled, a PodSecurityPolicy is created
pspEnabled: false
# ServiceMonitor configuration
serviceMonitor:
# -- If enabled, ServiceMonitor resources for Prometheus Operator are created
enabled: false
# -- Alternative namespace for ServiceMonitor resources
namespace: null
# -- Namespace selector for ServiceMonitor resources
namespaceSelector: {}
# -- ServiceMonitor annotations
annotations: {}
# -- Additional ServiceMonitor labels
labels: {}
# -- ServiceMonitor scrape interval
interval: null
# -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s)
scrapeTimeout: null
# -- ServiceMonitor relabel configs to apply to samples before scraping
# https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
relabelings: []
# -- ServiceMonitor will use http by default, but you can pick https as well
scheme: http
# -- ServiceMonitor will use these tlsConfig settings to make the health check requests
tlsConfig: null
# Rules for the Prometheus Operator
prometheusRule:
# -- If enabled, a PrometheusRule resource for Prometheus Operator is created
enabled: false
# -- Alternative namespace for the PrometheusRule resource
namespace: null
# -- PrometheusRule annotations
annotations: {}
# -- Additional PrometheusRule labels
labels: {}
# -- Contents of Prometheus rules file
groups: []
# - name: loki-rules
# rules:
# - record: job:loki_request_duration_seconds_bucket:sum_rate
# expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job)
# - record: job_route:loki_request_duration_seconds_bucket:sum_rate
# expr: sum(rate(loki_request_duration_seconds_bucket[1m])) by (le, job, route)
# - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate
# expr: sum(rate(container_cpu_usage_seconds_total[1m])) by (node, namespace, pod, container)
# Configuration for the ingester
ingester:
# -- Number of replicas for the ingester
replicas: 3
image:
# -- The Docker registry for the ingester image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the ingester image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the ingester image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for ingester pods
priorityClassName: null
# -- Annotations for ingester pods
podAnnotations: {}
# -- Labels for ingestor service
serviceLabels: {}
# -- Additional CLI args for the ingester
extraArgs: []
# -- Environment variables to add to the ingester pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the ingester pods
extraEnvFrom: []
# -- Volume mounts to add to the ingester pods
extraVolumeMounts: []
# -- Volumes to add to the ingester pods
extraVolumes: []
# -- Resource requests and limits for the ingester
resources: {}
# -- Grace period to allow the ingester to shutdown before it is killed. Especially for the ingestor,
# this must be increased. It must be long enough so ingesters can be gracefully shutdown flushing/transferring
# all data and to successfully leave the member ring on shutdown.
terminationGracePeriodSeconds: 300
# -- Affinity for ingester pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.ingesterSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.ingesterSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for ingester pods
nodeSelector: {}
# -- Tolerations for ingester pods
tolerations: []
persistence:
# Vaib - This si used by the boltdb shipper to save the indexes before flushing it to s3
# Vaib - https://github.com/grafana/loki/blob/main/docs/sources/operations/storage/boltdb-shipper.md
# -- Enable creating PVCs which is required when using boltdb-shipper
enabled: true
# -- Size of persistent disk
size: 50Gi
# -- Storage class to be used.
# If defined, storageClassName: <storageClass>.
# If set to "-", storageClassName: "", which disables dynamic provisioning.
# If empty or set to null, no storageClassName spec is
# set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
storageClass: null
# Configuration for the distributor
distributor:
# -- Number of replicas for the distributor
replicas: 1
autoscaling:
# -- Enable autoscaling for the distributor
enabled: true
# -- Minimum autoscaling replicas for the distributor
minReplicas: 3
# -- Maximum autoscaling replicas for the distributor
maxReplicas: 5
# -- Target CPU utilisation percentage for the distributor
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the distributor
targetMemoryUtilizationPercentage:
image:
# -- The Docker registry for the distributor image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the distributor image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the distributor image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for distributor pods
priorityClassName: null
# -- Annotations for distributor pods
podAnnotations: {}
# -- Labels for distributor service
serviceLabels: {}
# -- Additional CLI args for the distributor
extraArgs: []
# -- Environment variables to add to the distributor pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the distributor pods
extraEnvFrom: []
# -- Volume mounts to add to the distributor pods
extraVolumeMounts: []
# -- Volumes to add to the distributor pods
extraVolumes: []
# -- Resource requests and limits for the distributor
resources: {}
# -- Grace period to allow the distributor to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for distributor pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.distributorSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.distributorSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for distributor pods
nodeSelector: {}
# -- Tolerations for distributor pods
tolerations: []
# Configuration for the querier
# The querier service handles queries using the LogQL query language, fetching logs both from the ingesters and from long-term storage.
querier:
# -- Number of replicas for the querier
replicas: 1
autoscaling:
# -- Enable autoscaling for the querier, this is only used if `queryIndex.enabled: true`
enabled: true
# -- Minimum autoscaling replicas for the querier
minReplicas: 3
# -- Maximum autoscaling replicas for the querier
maxReplicas: 20
# -- Target CPU utilisation percentage for the querier
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the querier
targetMemoryUtilizationPercentage:
image:
# -- The Docker registry for the querier image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the querier image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the querier image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for querier pods
priorityClassName: null
# -- Annotations for querier pods
podAnnotations: {}
# -- Labels for querier service
serviceLabels: {}
# -- Additional CLI args for the querier
extraArgs: []
# -- Environment variables to add to the querier pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the querier pods
extraEnvFrom: []
# -- Volume mounts to add to the querier pods
extraVolumeMounts: []
# -- Volumes to add to the querier pods
extraVolumes: []
# -- Resource requests and limits for the querier
resources: {}
# -- Grace period to allow the querier to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for querier pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.querierSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.querierSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for querier pods
nodeSelector: {}
# -- Tolerations for querier pods
tolerations: []
persistence:
# -- Enable creating PVCs for the querier cache
enabled: true
# -- Size of persistent disk
size: 50Gi
# -- Storage class to be used.
# If defined, storageClassName: <storageClass>.
# If set to "-", storageClassName: "", which disables dynamic provisioning.
# If empty or set to null, no storageClassName spec is
# set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack).
storageClass: null
# Configuration for the query-frontend
queryFrontend:
# -- Number of replicas for the query-frontend
replicas: 1
autoscaling:
# -- Enable autoscaling for the query-frontend
enabled: true
# -- Minimum autoscaling replicas for the query-frontend
minReplicas: 2
# -- Maximum autoscaling replicas for the query-frontend
maxReplicas: 3
# -- Target CPU utilisation percentage for the query-frontend
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the query-frontend
targetMemoryUtilizationPercentage:
image:
# -- The Docker registry for the query-frontend image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the query-frontend image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the query-frontend image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for query-frontend pods
priorityClassName: null
# -- Annotations for query-frontend pods
podAnnotations: {}
# -- Labels for query-frontend service
serviceLabels: {}
# -- Additional CLI args for the query-frontend
extraArgs: []
# -- Environment variables to add to the query-frontend pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the query-frontend pods
extraEnvFrom: []
# -- Volume mounts to add to the query-frontend pods
extraVolumeMounts: []
# -- Volumes to add to the query-frontend pods
extraVolumes: []
# -- Resource requests and limits for the query-frontend
resources: {}
# -- Grace period to allow the query-frontend to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for query-frontend pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.queryFrontendSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.queryFrontendSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for query-frontend pods
nodeSelector: {}
# -- Tolerations for query-frontend pods
tolerations: []
# Configuration for the table-manager
tableManager:
# -- Specifies whether the table-manager should be enabled
enabled: false
image:
# -- The Docker registry for the table-manager image. Overrides `loki.image.registry`
registry: null
# -- Docker image repository for the table-manager image. Overrides `loki.image.repository`
repository: null
# -- Docker image tag for the table-manager image. Overrides `loki.image.tag`
tag: null
# -- The name of the PriorityClass for table-manager pods
priorityClassName: null
# -- Annotations for table-manager pods
podAnnotations: {}
# -- Labels for table-manager service
serviceLabels: {}
# -- Additional CLI args for the table-manager
extraArgs: []
# -- Environment variables to add to the table-manager pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the table-manager pods
extraEnvFrom: []
# -- Volume mounts to add to the table-manager pods
extraVolumeMounts: []
# -- Volumes to add to the table-manager pods
extraVolumes: []
# -- Resource requests and limits for the table-manager
resources: {}
# -- Grace period to allow the table-manager to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for table-manager pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.tableManagerSelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.tableManagerSelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for table-manager pods
nodeSelector: {}
# -- Tolerations for table-manager pods
tolerations: []
# Configuration for the gateway
gateway:
# -- Specifies whether the gateway should be enabled
enabled: true
# -- Number of replicas for the gateway
replicas: 1
autoscaling:
# -- Enable autoscaling for the gateway
enabled: false
# -- Minimum autoscaling replicas for the gateway
minReplicas: 1
# -- Maximum autoscaling replicas for the gateway
maxReplicas: 3
# -- Target CPU utilisation percentage for the gateway
targetCPUUtilizationPercentage: 60
# -- Target memory utilisation percentage for the gateway
targetMemoryUtilizationPercentage:
# -- See `kubectl explain deployment.spec.strategy` for more
# -- ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy
deploymentStrategy:
type: RollingUpdate
image:
# -- The Docker registry for the gateway image
registry: docker.io
# -- The gateway image repository
repository: nginxinc/nginx-unprivileged
# -- The gateway image tag
tag: 1.19-alpine
# -- The gateway image pull policy
pullPolicy: IfNotPresent
# -- The name of the PriorityClass for gateway pods
priorityClassName: null
# -- Annotations for gateway pods
podAnnotations: {}
# -- Additional CLI args for the gateway
extraArgs: []
# -- Environment variables to add to the gateway pods
extraEnv: []
# -- Environment variables from secrets or configmaps to add to the gateway pods
extraEnvFrom: []
# -- Volumes to add to the gateway pods
extraVolumes: []
# -- Volume mounts to add to the gateway pods
extraVolumeMounts: []
# -- The SecurityContext for gateway containers
podSecurityContext:
fsGroup: 101
runAsGroup: 101
runAsNonRoot: true
runAsUser: 101
# -- The SecurityContext for gateway containers
containerSecurityContext:
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
allowPrivilegeEscalation: false
# -- Resource requests and limits for the gateway
resources: {}
# -- Grace period to allow the gateway to shutdown before it is killed
terminationGracePeriodSeconds: 30
# -- Affinity for gateway pods. Passed through `tpl` and, thus, to be configured as string
# @default -- Hard node and soft zone anti-affinity
affinity: |
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
{{- include "loki.gatewaySelectorLabels" . | nindent 10 }}
topologyKey: kubernetes.io/hostname
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
{{- include "loki.gatewaySelectorLabels" . | nindent 12 }}
topologyKey: failure-domain.beta.kubernetes.io/zone
# -- Node selector for gateway pods
nodeSelector: {}
# -- Tolerations for gateway pods
tolerations: []
# Gateway service configuration
service:
# -- Port of the gateway service
port: 80
# -- Type of the gateway service
type: ClusterIP
# -- ClusterIP of the gateway service
clusterIP: null
# -- Node port if service type is NodePort
nodePort: null
# -- Load balancer IPO address if service type is LoadBalancer
loadBalancerIP: null
# -- Annotations for the gateway service
annotations: {}
# -- Labels for gateway service
labels: {}
# Gateway ingress configuration
ingress:
# -- Specifies whether an ingress for the gateway should be created
enabled: false
# -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18
# ingressClassName: nginx
# -- Annotations for the gateway ingress
annotations: {}
# -- Hosts configuration for the gateway ingress
hosts:
- host: gateway.loki.example.com
paths:
- path: /
# -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers
# pathType: Prefix
# -- TLS configuration for the gateway ingress
tls:
- secretName: loki-gateway-tls
hosts:
- gateway.loki.example.com
# Basic auth configuration
basicAuth:
# -- Enables basic authentication for the gateway
enabled: false
# -- The basic auth username for the gateway
username: null
# -- The basic auth password for the gateway
password: null
# -- Uses the specified username and password to compute a htpasswd using Sprig's `htpasswd` function.
# The value is templated using `tpl`. Override this to use a custom htpasswd, e.g. in case the default causes
# high CPU load.
htpasswd: >-
{{ htpasswd (required "'gateway.basicAuth.username' is required" .Values.gateway.basicAuth.username) (required "'gateway.basicAuth.password' is required" .Values.gateway.basicAuth.password) }}
# -- Existing basic auth secret to use. Must contain '.htpasswd'
existingSecret: null
# Configures the readiness probe for the gateway
readinessProbe:
httpGet:
path: /
port: http
initialDelaySeconds: 15
timeoutSeconds: 1
nginxConfig:
# -- NGINX log format
logFormat: |-
main '$remote_addr - $remote_user [$time_local] $status '
'"$request" $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
# -- Allows appending custom configuration to the server block
serverSnippet: ""
# -- Allows appending custom configuration to the http block
httpSnippet: ""
# -- Config file contents for Nginx. Passed through the `tpl` function to allow templating
# @default -- See values.yaml
file: |
worker_processes 5; ## Default: 1
error_log /dev/stderr;
pid /tmp/nginx.pid;
worker_rlimit_nofile 8192;
events {
worker_connections 4096; ## Default: 1024
}
http {
client_body_temp_path /tmp/client_temp;
proxy_temp_path /tmp/proxy_temp_path;
fastcgi_temp_path /tmp/fastcgi_temp;
uwsgi_temp_path /tmp/uwsgi_temp;
scgi_temp_path /tmp/scgi_temp;
default_type application/octet-stream;
log_format {{ .Values.gateway.nginxConfig.logFormat }}
access_log /dev/stderr main;
sendfile on;
tcp_nopush on;
resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }};
{{- with .Values.gateway.nginxConfig.httpSnippet }}
{{ . | nindent 2 }}
{{- end }}
server {
listen 8080;
{{- if .Values.gateway.basicAuth.enabled }}
auth_basic "Loki";
auth_basic_user_file /etc/nginx/secrets/.htpasswd;
{{- end }}
location = / {
return 200 'OK';
auth_basic off;
}
location = /api/prom/push {
proxy_pass http://{{ include "loki.distributorFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
}
location = /api/prom/tail {
proxy_pass http://{{ include "loki.querierFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location ~ /api/prom/.* {
proxy_pass http://{{ include "loki.queryFrontendFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
}
location = /loki/api/v1/push {
proxy_pass http://{{ include "loki.distributorFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
}
location = /loki/api/v1/tail {
proxy_pass http://{{ include "loki.querierFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location ~ /loki/api/.* {
proxy_pass http://{{ include "loki.queryFrontendFullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:3100$request_uri;
}
{{- with .Values.gateway.nginxConfig.serverSnippet }}
{{ . | nindent 4 }}
{{- end }}
}
}
Installed Promtail using the official chart:
https://github.com/grafana/helm-charts/tree/main/charts/promtail
Values.yaml file:
scrapeConfigs: |
# See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
- job_name: kubernetes-pods-app-kubernetes-io-name
# Kubernets auto discovery pods
kubernetes_sd_configs:
- role: pod
# Replacing kubernetes.io.name with app
relabel_configs:
- action: replace
source_labels:
- __meta_kubernetes_pod_label_app_kubernetes_io_name
target_label: app
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: pod
# Dropping logs that does not have app label
- action: drop
regex: ''
source_labels:
- app
# Replace namespace with namespace
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
# Required by promtail config to tail the pod logs
- action: replace
replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
# Removing helath and ping logs from the application
pipeline_stages:
- match:
selector: '{app=~".*"} |~ ".*/(health|ping).*"'
action: drop
drop_counter_reason: promtail_noisy_error
- match:
selector: '{namespace="loki"}'
action: drop
drop_counter_reason: dont_like_it
- match:
selector: '{app=~".*"}'
stages:
- template:
source: pod
template: '{ "pod": {{ `"{{ .Value }}"` }}, "app": {{ `"{{ .app }}"` }} ,"pod_log": {{ `{{ .Entry }}` }} }'
- output:
source: pod
- labeldrop:
- filename
- pod
- app
# -- Config file contents for Promtail.
# Must be configured as string.
# It is templated so it can be assembled from reusable snippets in order to avoid redundancy.
# @default -- See `values.yaml`
file: |
server:
log_level: {{ .Values.config.logLevel }}
http_listen_port: {{ .Values.config.serverPort }}
client:
url: {{ tpl .Values.config.lokiAddress . }}
{{- tpl .Values.config.snippets.extraClientConfigs . | nindent 2 }}
positions:
filename: /run/promtail/positions_ash.yaml
There are many issues that I am encountering:
- When I am trying to add the Loki source in grafana(gateway endpoint)
Grafana is throwing error:
Loki: Internal Server Error. 500. rpc error: code = Unavailable desc = name resolver error: produced zero addresses
- Distributer logs showing:
level=warn ts=2021-10-06T10:41:23.729163833Z caller=grpc_logging.go:38 method=/logproto.Pusher/Push duration=86.664µs err=“rpc error: code = Code(400) desc = entry with timestamp 2021-10-06 10:41:22.689555007 +0000 UTC ignored, reason: ‘entry out of order’ for stream: {namespace="integration"},\nentry with timestamp 2021-10-06 10:41:22.689562602 +0000 UTC ignored, reason: ‘entry out of order’ for stream: {namespace="integration"},\nentry with timestamp 2021-10-06 10:41:22.689582704 +0000 UTC ignored, reason: ‘entry out of order’ for stream: {namespace="integration"},\ntotal ignored: 3 out of 6” msg=“gRPC\n”
I am not sure what I am doing wrong. I tried all possible changes in the values file.
I suspect there is something wrong with the Loki configuration file.
Note: When I try installing the loki and promtail with default values.yaml file it works but I can still see the error.
reason: ‘entry out of order’