add local o11y

Author: abbiesims

.env

@@ -4,3 +4,11 @@ FLASK_DEBUG=true
 DEVEL=True
 SECRET_KEY=local_development_fake_key
 LP_API_USERNAME=test_lp_user
+
+# OpenTelemetry
+# Uncomment to enable tracing:
+OTEL_SERVICE_NAME=snapcraft-io
+# if using Linux, uncomment this:
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+# if using MacOS, uncomment this:
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:4318

observability/README.md

@@ -0,0 +1,69 @@
+# Local Tracing for charmhub.io
+
+This guide explains how to test OpenTelemetry traces locally using OpenTelemetry, Grafana and Tempo. 
+
+*This setup is intended for local development and testing purposes only.*
+
+## Overview
+
+This setup enables observability for local development by capturing traces from the application and visualising them in Grafana, using Tempo as the backend.
+
+## Prerequisites
+
+- Docker and Docker Compose installed
+- A properly configured `.env` file
+
+## Setup Instructions
+
+### 1. Configure environment variables
+
+In your `.env` file, make the following changes:
+
+- Uncomment the appropriate `OTEL_EXPORTER_OTLP_ENDPOINT` based on your operating system.
+- Uncomment `OTEL_SERVICE_NAME` to enable tracing.
+
+### 2. Start observability stack
+
+Open a new terminal window and run:
+
+`cd observability`
+`docker compose up -d`
+
+This starts the OpenTelemetry Collector, Grafana and Tempo containers for tracing.
+
+### 3. Run the application
+
+Back in the main terminal (in the root of the project), run `dotrun`
+
+### 4. Generate traces
+
+Interact with the application by visiting various pages such as:
+
+- Homepage (list of charms)
+- Charm pages
+- Login page
+- Publisher pages
+
+These interactions will generate traces.
+
+### 5. View traces in Grafana
+
+Open Grafana in the browser at: http://localhost:3000
+
+Login using the default credentials:
+
+- Username: `admin`
+- Password: `admin`
+
+Then:
+1. Go to `Explore`
+2. The Tempo datasource should be selected
+3. Click `Search`
+4. You should see a list of trace IDs
+5. Click on a trace ID to view detailed nested spans
+
+### 6. Stop the observability stack
+Once you're done testing, you can shut down the containers:
+`cd observability`
+`docker compose down`
+

observability/docker-compose.yaml

@@ -0,0 +1,46 @@
+services:
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
+    depends_on:
+      - tempo
+    networks:
+      - observability
+
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    networks:
+      - observability
+
+  tempo:
+    image: grafana/tempo:latest
+    ports:
+      - "3200:3200"  # Tempo UI/query
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./tempo/tempo.yaml:/etc/tempo.yaml
+    networks:
+      - observability
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: [ "--config=/etc/otel-collector-config.yaml" ]
+    ports:
+      - "4318:4318"  # OTLP HTTP - app on localhost sends traces here
+    volumes:
+      - ./otel-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    depends_on:
+      - tempo
+    networks:
+      - observability
+
+networks:
+  observability:
+    driver: bridge

observability/grafana/provisioning/datasources/datasource.yaml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+datasources:
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    isDefault: true
+  
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090

observability/otel-collector/otel-collector-config.yaml

@@ -0,0 +1,17 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "0.0.0.0:4318"
+
+exporters:
+  otlp:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [otlp]

observability/prometheus/prometheus.yml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+  - job_name: 'snapcraft-io'
+    metrics_path: '/_status/metrics'
+    static_configs:
+      - targets: ['snapcraft.io']

observability/tempo/tempo.yaml

@@ -0,0 +1,25 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 5_000_000
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo/traces

requirements.txt

@@ -30,3 +30,13 @@ Flask-Testing==0.8.1
 freezegun==1.4.0
 black==24.3.0
 flake8==6.1.0
+
+# Observability
+opentelemetry-api==1.32.1
+opentelemetry-exporter-otlp==1.32.1
+opentelemetry-exporter-otlp-proto-http==1.32.1
+opentelemetry-instrumentation==0.53b1
+opentelemetry-instrumentation-wsgi==0.53b1
+opentelemetry-instrumentation-flask==0.53b1
+opentelemetry-instrumentation-requests==0.53b1
+opentelemetry-sdk==1.32.1

webapp/app.py

@@ -26,10 +26,44 @@
 from webapp.tutorials.views import init_tutorials
 from webapp.packages.store_packages import store_packages
 
+# --- OpenTelemetry imports ---
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.trace import Span
+from webapp.observability.utils import trace_function
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+    OTLPSpanExporter
+)
+from opentelemetry.sdk.resources import Resource
 
 TALISKER_WSGI_LOGGER = logging.getLogger("talisker.wsgi")
 
 
+# OpenTelemetry
+UNTRACED_ROUTES = [
+    "/_status",
+    ".*[.jpg|.jpeg|.png|.gif|.ico|.css|.js|.json]$",
+]
+
+
+# Setup tracing manually
+# could be removed if we can run dotrun with opentelemetry
+resource = Resource.create()
+trace.set_tracer_provider(TracerProvider(resource=resource))
+tracer_provider = trace.get_tracer_provider()
+otlp_exporter = OTLPSpanExporter()  # reads env variables
+tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
+
+
+@trace_function
+def request_hook(span: Span, environ):
+    if span and span.is_recording():
+        span.update_name(f"{environ['REQUEST_METHOD']} {environ['PATH_INFO']}")
+
+
 def create_app(testing=False):
     app = FlaskBase(
         __name__,
@@ -49,6 +83,13 @@ def create_app(testing=False):
         talisker.requests.configure(webapp.helpers.api_session)
         talisker.requests.configure(webapp.helpers.api_publisher_session)
 
+        # Add tracing auto instrumentation
+        FlaskInstrumentor().instrument_app(
+            app, excluded_urls=",".join(UNTRACED_ROUTES),
+            request_hook=request_hook
+        )
+        RequestsInstrumentor().instrument()
+
     if testing:
 
         @app.context_processor

webapp/helpers.py

@@ -8,6 +8,7 @@
 from webapp.api.requests import PublisherSession, Session
 from canonicalwebteam.store_api.dashboard import Dashboard
 import webapp.api.marketo as marketo_api
+from webapp.observability.utils import trace_function
 
 _yaml = YAML(typ="rt")
 _yaml_safe = YAML(typ="safe")
@@ -98,13 +99,15 @@ def dump_yaml(data, stream, typ="safe"):
     yaml.dump(data, stream)
 
 
+@trace_function
 def get_icon(media):
     icons = [m["url"] for m in media if m["type"] == "icon"]
     if len(icons) > 0:
         return icons[0]
     return ""
 
 
+@trace_function
 def get_publisher_data():
     # We don't use the data from this endpoint.
     # It is mostly used to make sure the user has signed
@@ -139,6 +142,7 @@ def get_publisher_data():
     return context
 
 
+@trace_function
 def get_dns_verification_token(snap_name, domain):
     salt = os.getenv("DNS_VERIFICATION_SALT")
     token_string = f"{domain}:{snap_name}:{salt}"