Grafana Agent + Tempo + OpenTelemetry (Node.js) not sending traces

Grafana Tempo
1

Hello,

I currently set up Grafana Tempo and Grafana Agent for shipping Traces sent by a Node.js application utilizing opentelemetry instrumentation.
For some reason the traces never seem to reach Tempo but there are no error logs and I can’t seem to figure out what the problem is.
This is my Grafana Agent config (Terraform Code):

server = {
   log_level = "debug"
 }
 traces = {
      configs = [{
        batch = {
          send_batch_size = 1000
          timeout         = "5s"
        }
        name = "tempo-grafana-agent"
        receivers = {
          zipkin = null
          otlp = {
            protocols = {
              grpc = null
              http = null
            }
          }
        }
        remote_write = [{
          protocol = "http"
          endpoint = var.traces_client_url
          retry_on_failure = {
            enabled = true
          }
        }]
        scrape_configs = [{
          bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
          job_name          = "kubernetes-pods"
          kubernetes_sd_configs = [{
            role = "pod"
          }]
          relabel_configs = [
            {
              action = "replace"
              source_labels = [
                "__meta_kubernetes_namespace"
              ]
              target_label = "namespace"
            },
            {
              action = "replace"
              source_labels = [
                "__meta_kubernetes_pod_name"
              ]
              target_label = "pod"
            },
            {
              action = "replace"
              source_labels = [
                "__meta_kubernetes_pod_container_name"
              ]
              target_label = "container"
          }]
          tls_config = {
            ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
            insecure_skip_verify = false
          }
        }]
      }]
    }
  }
  strategies = {
    "default_strategy" = {
      "param" = 0.001
      "type"  = "probabilistic"
    }
  }

where traces_client_url is http://tempo-distributed-distributor.${kubernetes_namespace_v1.tempo.metadata.0.name}.svc.cluster.local:4318/v1/traces

The simplistic instrumentation in Node.js is set up like this:

/*instrumentation.js*/
// Require dependencies
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { PeriodicExportingMetricReader, ConsoleMetricExporter } = require('@opentelemetry/sdk-metrics');

process.env.OTEL_LOG_LEVEL = 'debug';

const sdk = new NodeSDK({
  traceExporter: new OTLPTraceExporter({
    url: "http://grafana-agent-traces.scraper.svc.cluster.local:55680",
    headers: {},
  }),
  metricReader: new PeriodicExportingMetricReader({
    exporter: new ConsoleMetricExporter()
  }),
  instrumentations: [getNodeAutoInstrumentations()]
});

sdk
  .start()

I tried to resolve the issue by following this article: Unable to find traces | Grafana Tempo documentation
But it seems like although metaMonitoring is configured for Tempo (I’m using tempo-distributed helm chart) the metrics mentioned in this article are not available.

Any idea what could be the problem in my configuration?

Thanks in advance for any hint or help!

Greetings!

PS: solved it, apparently I missed that agent’s endpoint is /v1/traces

1 Like
2

glad to hear that it worked, will look into docs and see if we can hightlight the fact that endpoint needs /v1/traces.

3

This topic was automatically closed 365 days after the last reply. New replies are no longer allowed.