Loki Behind reverse proxy 404

[giovannihenrique1989]

Hi there,

I’m facing a really annoying error when setting up my grafana stack server.
I have a nginx reverse proxying for multiple applications that I’ve setted up. here’s my proxy.conf:
upstream grafana {
server localhost:3000;
}

upstream loki {
server localhost:3100;
}

upstream prometheus {
server localhost:9090;
}

upstream blackbox {
server localhost:9115;
}

upstream node {
server localhost:9100;
}

#Redirects HTTP to HTTPS
server {
listen 80;
listen [::]:80;
server_name <mydomainname.provider>;

location ~* ^\/grafana\/loki\/*[.+]* {
    proxy_http_version 1.1;
    proxy_set_header Host $http_host;
    proxy_pass http://loki;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Connection "Keep-Alive";
    proxy_set_header Proxy-Connection "Keep-Alive";
    proxy_redirect off;
}

location / {
return 301 https://$server_name$request_uri;
}

}

#HTTPS Server
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name <mydomainname.provider>;
ssl_certificate /etc/ssl/certs/mydomainname.pem;
ssl_certificate_key /etc/ssl/certs/mydomainname.pem;
ssl_client_certificate /etc/ssl/certs/mydomainname.pem;
ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;
ssl_verify_client optional;

location /prometheus/blackbox/ {
    proxy_set_header Host $http_host;
    proxy_pass http://blackbox;
#     include proxy_params;
}

location /prometheus/node/ {
    proxy_set_header Host $http_host;
    proxy_pass http://node;
#     include proxy_params;
}

location /prometheus {
    proxy_set_header Host $http_host;
    proxy_pass http://prometheus;
}

location ~* ^\/grafana\/loki\/*[.+]* {
    proxy_http_version 1.1;
    proxy_set_header Host $http_host;
    proxy_pass http://loki;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Connection "Keep-Alive";
    proxy_set_header Proxy-Connection "Keep-Alive";
    proxy_redirect off;
}

location /grafana/api/live/ {
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $http_host;
    proxy_pass http://grafana;
}

location /grafana {
    proxy_set_header Host $http_host;
    proxy_pass http://grafana;
}


location / {
    return 301 /grafana/;
}

}

map $http_upgrade $connection_upgrade {
default upgrade;
‘’ close;
}

When I check server logs for errors, no apparent error shows up, although, when trying to add the loki datasource to grafana, I’m facing a 404 error:

I’ve tried a different proxy settings but none worked for me. I’m able to access all loki APIs and even my promtail is being able to push logs to it, returning 204.

Here’s my loki server conf:
auth_enabled: false

server:
http_listen_network: tcp
http_listen_address: localhost
http_listen_port: 3100
http_path_prefix: /grafana/loki
grpc_listen_port: 9096

common:
instance_addr: 127.0.0.1
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory

query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100

schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h

analytics:
reporting_enabled: false

Here’s my promtail conf:
server:
http_listen_port: 9080
grpc_listen_port: 0

positions:
filename: /etc/grafana/promtail/temp/positions.yaml

clients:

• url: https://mydomain.provider/grafana/loki/loki/api/v1/push

scrape_configs:

• job_name: grafana
  static_configs:

  • targets:
    • localhost
      labels:
      job: nginx
      path: /var/log/nginx/grafana/*log

• job_name: nginx
  static_configs:

  • targets:
    • localhost
      labels:
      job: nginx
      path: /var/log/nginx/*log

• job_name: system
  static_configs:

  • targets:
    • localhost
      labels:
      job: varlogs
      path: /var/log/*log

I don’t know exaclty what I’m messing up, everything looks working, but when trying to add to grafana, I’m getting a 404. Anyone can help?

Nginx version: 1.24
Loki version: 2.8.x
Promtail version: 2.8.2
Grafana version: 10.0.0

Edit: I’m NOT running on docker