FS: Add tempo tracing to dev stack (#109682)

* add tempo

* trying to debug trace propagation

* add extra header

* change header to inject

* very debug logging

* set GF_TRACING_OPENTELEMETRY_OTLP_PROPAGATION

* clean up config

* undo temp debug logging

devenv/frontend-service/Tiltfile

@@ -69,6 +69,8 @@ dc_resource("frontend-service",
 )
 dc_resource("alloy", labels=["observability"])
 dc_resource("loki", labels=["observability"])
+dc_resource("tempo-init", labels=["observability"])
+dc_resource("tempo", labels=["observability"])
 
 # paths in tilt files are confusing....
 # - if tilt is dealing the the path, it is relative to the Tiltfile
@@ -111,10 +113,10 @@ docker_build('grafana-proxy',
 
   # Path relative to the docker context (this folder)
   only=[
-    "./nginx.conf",
+    "./configs/nginx.conf",
   ],
   live_update = [
-    sync('./nginx.conf', '/etc/nginx/conf.d/default.conf'),
+    sync('./configs/nginx.conf', '/etc/nginx/conf.d/default.conf'),
     restart_container()
   ]
 )

devenv/frontend-service/configs/alloy/_databases.alloy

@@ -1,3 +1,5 @@
+//
+// Logs
 loki.relabel "publish_logs" {
 	rule {
 		action       = "replace"
@@ -13,3 +15,31 @@ loki.write "tilt_loki" {
 		url = "http://loki:3100/loki/api/v1/push"
 	}
 }
+
+//
+// Traces
+otelcol.processor.transform "publish_traces" {
+	error_mode = "ignore"
+
+	trace_statements {
+		context    = "resource"
+		statements = [
+			string.format(`set(attributes["instance"], "%s")`, constants.hostname),
+		]
+	}
+
+	output {
+		traces = [otelcol.exporter.otlp.tilt_tempo.input]
+	}
+}
+
+otelcol.exporter.otlp "tilt_tempo" {
+	client {
+		endpoint = "tempo:4317"
+
+		tls {
+			insecure             = true
+			insecure_skip_verify = true
+		}
+	}
+}

devenv/frontend-service/configs/alloy/otel-receiver.alloy

@@ -0,0 +1,13 @@
+//
+// Traces
+otelcol.receiver.otlp "grafana_dev" {
+	output {
+		metrics = []
+		logs    = []
+		traces  = [otelcol.processor.transform.publish_traces.input]
+	}
+
+	grpc {
+		endpoint = "0.0.0.0:4317"
+	}
+}

devenv/frontend-service/configs/nginx.conf

@@ -1,3 +1,9 @@
+otel_exporter {
+  endpoint alloy:4317;
+}
+
+otel_service_name "proxy";
+
 ###
 # Instance
 ###
@@ -14,10 +20,19 @@ map "$request_method:$cookie_fs_unavailable" $reject_login {
   "POST:1" 1;
 }
 
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
 server {
   listen 80;
   server_name _;
 
+  otel_trace         on;
+  otel_trace_context inject;
+  otel_span_name "$request_method Request";
+
   location ~ ^/-/down/?$ {
     add_header Set-Cookie "fs_unavailable=true; Max-Age=60; Path=/; HttpOnly" always;
     return 302 $scheme://$http_host/;
@@ -58,9 +73,11 @@ server {
   }
 
   # API calls go to the backend
-  # Cheat with app plugin paths and route them to the backend. These should come from
-  # the Plugin CDN
   location ~ ^/(api|apis|avatar|bootdata|render|logout|public\/plugins) {
+    # Add debug headers to the response
+    add_header Nginx-Trace-Id $otel_trace_id always;
+    add_header Nginx-Route "backend" always;
+
     if ($cookie_fs_unavailable) {
       add_header Content-Type application/json always;
       return 503 '{"code":"Loading", "message": "Soon!"}';
@@ -71,12 +88,18 @@ server {
     proxy_set_header X-Real-IP         $remote_addr;
     proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
     proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header Upgrade    $http_upgrade;
+
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
   }
 
   # Everything else to the frontend
   location / {
+    # Add debug headers to the response
+    add_header Nginx-Trace-Id $otel_trace_id always;
+    add_header Nginx-Route "frontend" always;
+
     proxy_pass http://frontend;
     proxy_set_header Host              $host;
     proxy_set_header X-Real-IP         $remote_addr;

devenv/frontend-service/configs/tempo.yaml

@@ -0,0 +1,48 @@
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+  log_level: info
+
+distributor:
+  usage:
+    cost_attribution:
+      enabled: true
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 'tempo:4317'
+        http:
+          endpoint: 'tempo:4318'
+
+storage:
+  trace:
+    backend: local # Specify local storage explicitly
+
+    # The Write-Ahead Log (WAL) is a logging mechanism used to ensure data integrity. In the context of Tempo,
+    # the WAL is used to record changes to trace data before they are committed to the main storage.
+    #This helps in recovering data in case of a failure. The WAL stores these changes in a sequential log file,
+    #which can be replayed to reconstruct the state of the system.
+    wal:
+      path: /var/lib/tempo/wal
+    # indicates that the storage backend is local, meaning the data will be stored on the local filesystem.
+    local:
+      path: /var/lib/tempo/blocks # Path for block storage
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/lib/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/lib/tempo/generator/traces
+  processor:
+    local_blocks:
+      filter_server_spans: false
+      flush_to_storage: true

devenv/frontend-service/docker-compose.yaml

@@ -30,15 +30,18 @@ services:
       - ./provisioning/dashboards:/grafana/conf/provisioning/dashboards
       - ../dev-dashboards:/grafana/conf/dev-dashboards
     environment:
+      OTEL_BSP_SCHEDULE_DELAY: 500
       GF_DEFAULT_APP_MODE: development
       GF_PANELS_ENABLE_ALPHA: true
       GF_SERVER_CDN_URL: http://localhost:3010
       GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend
-      GF_DATABASE_TYPE: postgres
+      GF_DATABASE_URL: postgres://grafana:grafana@postgres:5432/grafana
-      GF_DATABASE_HOST: postgres
+      GF_SERVER_ROUTER_LOGGING: true
-      GF_DATABASE_NAME: grafana
+      GF_LOG_LEVEL: info
-      GF_DATABASE_USER: grafana
+      GF_AUTH_LOGIN_COOKIE_NAME: grafana_fs_dev_login # set a custom cookie name to not conflict with other instances running on localhost
-      GF_DATABASE_PASSWORD: grafana
+      OTEL_SERVICE_NAME: grafana-api
+      GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: 'alloy:4317'
+      GF_TRACING_OPENTELEMETRY_OTLP_PROPAGATION: jaeger,w3c
     ports:
       - '3011:3000'
     labels:
@@ -55,10 +58,16 @@ services:
     labels:
       - 'alloy.logs=true'
     environment:
+      OTEL_BSP_SCHEDULE_DELAY: 500
       GF_DEFAULT_APP_MODE: development
       GF_DEFAULT_TARGET: frontend-server
       GF_SECURITY_CONTENT_SECURITY_POLICY: false
       GF_SERVER_CDN_URL: http://localhost:3010
+      GF_SERVER_ROUTER_LOGGING: true
+      GF_LOG_LEVEL: info
+      OTEL_SERVICE_NAME: frontend-service
+      GF_TRACING_OPENTELEMETRY_OTLP_ADDRESS: 'alloy:4317'
+      GF_TRACING_OPENTELEMETRY_OTLP_PROPAGATION: jaeger,w3c
 
   postgres:
     image: postgres:16.1-alpine3.19
@@ -95,8 +104,30 @@ services:
     labels:
       - 'alloy.logs=true'
 
+  tempo-init:
+    image: grafana/tempo
+    user: root
+    entrypoint:
+      - 'chown'
+      - '10001:10001'
+      - '/var/tempo'
+    volumes:
+      - tempo-data:/var/tempo
+
+  tempo:
+    image: grafana/tempo
+    volumes:
+      - tempo-data:/var/lib/tempo
+      - ./configs/tempo.yaml:/etc/tempo/tempo.yaml
+    command: ['-config.file=/etc/tempo/tempo.yaml']
+
+    depends_on:
+      tempo-init:
+        condition: service_completed_successfully
+
 volumes:
   backend-data:
   postgres-data:
   alloy-data:
   loki-data:
+  tempo-data:

devenv/frontend-service/provisioning/datasources/default.yaml

@@ -12,3 +12,16 @@ datasources:
     uid: fs-loki
     type: loki
     url: http://loki:3100
+
+  - name: Tempo Traces
+    uid: fs-tempo
+    type: tempo
+    url: http://tempo:3200
+    jsonData:
+      tracesToLogsV2:
+        # Field with an internal link pointing to a logs data source in Grafana.
+        # datasourceUid value must match the uid value of the logs data source.
+        datasourceUid: 'fs-loki'
+        spanStartTimeShift: '-1m'
+        spanEndTimeShift: '1m'
+        filterByTraceID: true

devenv/frontend-service/proxy.dockerfile

@@ -1,5 +1,18 @@
-# Runtime stage - just a simple nginx to serve static files
+FROM nginx:1.29.0-alpine
-# We bake the config into the image so we can live-update it with Tilt when it changes
-FROM nginx:alpine
 
-COPY nginx.conf /etc/nginx/conf.d/default.conf
+RUN apk add --no-cache openssl curl ca-certificates
+
+RUN printf "%s%s%s%s\n" \
+    "@nginx " \
+    "http://nginx.org/packages/mainline/alpine/v" \
+    `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` \
+    "/main" \
+    | tee -a /etc/apk/repositories
+
+RUN curl -o /tmp/nginx_signing.rsa.pub https://nginx.org/keys/nginx_signing.rsa.pub
+RUN mv /tmp/nginx_signing.rsa.pub /etc/apk/keys/
+RUN apk add --no-cache nginx-module-otel@nginx --force-overwrite
+
+RUN sed -i '1iload_module modules/ngx_otel_module.so;' /etc/nginx/nginx.conf
+
+COPY configs/nginx.conf /etc/nginx/conf.d/default.conf