feat: add distributed tracing for webhook handling and PipelineRun timing

Add OpenTelemetry distributed tracing to PaC. Spans cover webhook
event processing and PipelineRun lifecycle timing. Attribute naming
follows OTel semconv where possible (vcs.*, cicd.*, k8s.*), matches
Tekton's existing bare attribute names for resource identity, and
uses delivery.tekton.dev.* for domain-specific delivery pipeline
concepts per OTel reverse-domain naming guidance.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 2 people

cmd/pipelines-as-code-controller/main.go

@@ -40,7 +40,9 @@ func main() {
 	loggerConfigurator := evadapter.NewLoggerConfiguratorFromConfigMap(PACControllerLogKey, loggerConfiguratorOpt)
 	copts := []evadapter.ConfiguratorOption{
 		evadapter.WithLoggerConfigurator(loggerConfigurator),
-		evadapter.WithObservabilityConfigurator(evadapter.NewObservabilityConfiguratorFromConfigMap()),
+		evadapter.WithObservabilityConfigurator(evadapter.NewObservabilityConfiguratorFromConfigMap(
+			evadapter.WithObservabilityConfiguratorConfigMapName("pipelines-as-code-config-observability"),
+		)),
 		evadapter.WithCloudEventsStatusReporterConfigurator(evadapter.NewCloudEventsReporterConfiguratorFromConfigMap()),
 	}
 	// put logger configurator to ctx

config/305-config-observability.yaml

@@ -50,4 +50,18 @@ data:
 
     # metrics-export-interval specifies how often metrics are exported.
     # Only applicable for grpc and http/protobuf protocols.
-    # metrics-export-interval: "30s"
+    # metrics-export-interval: "30s"
+
+    # tracing-protocol specifies the trace export protocol.
+    # Supported values: "grpc", "http/protobuf", "none".
+    # Default is "none" (tracing disabled).
+    # tracing-protocol: "none"
+
+    # tracing-endpoint specifies the OTLP collector endpoint.
+    # Required when tracing-protocol is "grpc" or "http/protobuf".
+    # The OTEL_EXPORTER_OTLP_ENDPOINT env var takes precedence if set.
+    # tracing-endpoint: "http://otel-collector.observability.svc.cluster.local:4317"
+
+    # tracing-sampling-rate controls the fraction of traces sampled.
+    # 0.0 = none, 1.0 = all. Default is 0 (none).
+    # tracing-sampling-rate: "1.0"

docs/content/docs/operations/tracing.md

@@ -0,0 +1,51 @@
+---
+title: Distributed Tracing
+weight: 5
+---
+
+This page describes how to enable OpenTelemetry distributed tracing for Pipelines-as-Code. When enabled, PaC emits trace spans for webhook event processing and PipelineRun lifecycle timing.
+
+## Enabling tracing
+
+The ConfigMap `pipelines-as-code-config-observability` controls tracing configuration. See [config/305-config-observability.yaml](https://github.com/tektoncd/pipelines-as-code/blob/main/config/305-config-observability.yaml) for the full example.
+
+It contains the following tracing fields:
+
+* `tracing-protocol`: Export protocol. Supported values: `grpc`, `http/protobuf`, `none`. Default is `none` (tracing disabled).
+* `tracing-endpoint`: OTLP collector endpoint. Required when protocol is not `none`. The `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable takes precedence if set.
+* `tracing-sampling-rate`: Fraction of traces to sample. `0.0` = none, `1.0` = all. Default is `0`.
+
+### Example
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pipelines-as-code-config-observability
+  namespace: pipelines-as-code
+data:
+  tracing-protocol: grpc
+  tracing-endpoint: "http://otel-collector.observability.svc.cluster.local:4317"
+  tracing-sampling-rate: "1.0"
+```
+
+Changes to the ConfigMap are picked up automatically without restarting the controller. Set `tracing-protocol` to `none` or remove the tracing keys to disable tracing.
+
+## Emitted spans
+
+The controller emits a `PipelinesAsCode:ProcessEvent` span covering the full lifecycle of each webhook event, from receipt through PipelineRun creation. The watcher emits `waitDuration` and `executeDuration` spans for completed PipelineRuns, using the PipelineRun's actual timestamps for accurate wall-clock timing.
+
+## Trace context propagation
+
+When Pipelines-as-Code creates a PipelineRun, it sets the `tekton.dev/pipelinerunSpanContext` annotation with a JSON-encoded OTel TextMapCarrier containing the W3C `traceparent`. PaC tracing works independently — you get PaC spans regardless of whether Tekton Pipelines has tracing enabled.
+
+If Tekton Pipelines is also configured with tracing pointing at the same collector, its reconciler spans appear as children of the PaC span, providing a single end-to-end trace from webhook receipt through task execution. See the [Tekton Pipelines tracing documentation](https://github.com/tektoncd/pipeline/blob/main/docs/developers/tracing.md) for Tekton's independent tracing setup.
+
+## Deploying a trace collector
+
+Pipelines-as-Code exports traces using the standard OpenTelemetry Protocol (OTLP). You need a running OTLP-compatible collector for the `tracing-endpoint` to point to. Common options include:
+
+* [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) -- the vendor-neutral reference collector
+* [Jaeger](https://www.jaegertracing.io/docs/latest/getting-started/) -- supports OTLP ingestion natively since v1.35
+
+Deploying and operating a collector is outside the scope of Pipelines-as-Code. Refer to your organization's observability infrastructure or the links above for setup instructions.

go.mod

@@ -30,7 +30,9 @@ require (
 	gitlab.com/gitlab-org/api/client-go v1.46.0
 	go.opentelemetry.io/otel v1.42.0
 	go.opentelemetry.io/otel/metric v1.42.0
+	go.opentelemetry.io/otel/sdk v1.42.0
 	go.opentelemetry.io/otel/sdk/metric v1.42.0
+	go.opentelemetry.io/otel/trace v1.42.0
 	go.uber.org/zap v1.27.1
 	golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90
 	golang.org/x/oauth2 v0.36.0
@@ -91,8 +93,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 // indirect
 	go.opentelemetry.io/otel/exporters/prometheus v0.64.0 // indirect
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
-	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect

pkg/adapter/adapter.go

@@ -23,6 +23,10 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/gitea"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/github"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/gitlab"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/tracing"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"knative.dev/eventing/pkg/adapter/v2"
@@ -191,6 +195,19 @@ func (l listener) handleEvent(ctx context.Context) http.HandlerFunc {
 		}
 		gitProvider.SetPacInfo(&pacInfo)
 
+		// Extract inbound trace context from request headers for distributed tracing
+		tracedCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(request.Header))
+
+		// Start a span for webhook handling
+		tracer := otel.Tracer(tracing.TracerName)
+		tracedCtx, span := tracer.Start(tracedCtx, "PipelinesAsCode:ProcessEvent",
+			trace.WithSpanKind(trace.SpanKindServer),
+		)
+
+		span.SetAttributes(
+			tracing.VCSProviderName.String(gitProvider.GetConfig().Name),
+		)
+
 		s := sinker{
 			run:        l.run,
 			vcx:        gitProvider,
@@ -206,8 +223,10 @@ func (l listener) handleEvent(ctx context.Context) http.HandlerFunc {
 		localRequest := request.Clone(request.Context())
 
 		go func() {
-			err := s.processEvent(ctx, localRequest)
+			defer span.End()
+			err := s.processEvent(tracedCtx, localRequest)
 			if err != nil {
+				span.RecordError(err)
 				logger.Errorf("an error occurred: %v", err)
 			}
 		}()

pkg/adapter/sinker.go

@@ -14,6 +14,8 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/pipelineascode"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/status"
+	"github.com/openshift-pipelines/pipelines-as-code/pkg/tracing"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 )
 
@@ -117,6 +119,10 @@ func (s *sinker) processEvent(ctx context.Context, request *http.Request) error
 		}
 	}
 
+	// Enrich span with VCS attributes — for incoming events these are
+	// pre-populated; for webhook events ParsePayload filled them in.
+	setVCSSpanAttributes(ctx, s.event)
+
 	p := pipelineascode.NewPacs(s.event, s.vcx, s.run, s.pacInfo, s.kint, s.logger, s.globalRepo)
 	return p.Run(ctx)
 }
@@ -174,3 +180,18 @@ func (s *sinker) createSkipCIStatus(ctx context.Context) error {
 
 	return nil
 }
+
+// setVCSSpanAttributes enriches the current span with VCS attributes from the event.
+func setVCSSpanAttributes(ctx context.Context, event *info.Event) {
+	span := trace.SpanFromContext(ctx)
+	if !span.IsRecording() {
+		return
+	}
+	span.SetAttributes(tracing.VCSEventType.String(event.EventType))
+	if event.URL != "" {
+		span.SetAttributes(tracing.VCSRepositoryURLFull.String(event.URL))
+	}
+	if event.SHA != "" {
+		span.SetAttributes(tracing.VCSRefHeadRevision.String(event.SHA))
+	}
+}

pkg/apis/pipelinesascode/keys/keys.go

@@ -68,6 +68,9 @@ const (
 	GithubApplicationID  = "github-application-id"
 	GithubPrivateKey     = "github-private-key"
 	ResultsRecordSummary = "results.tekton.dev/recordSummaryAnnotations"
+
+	// SpanContextAnnotation is the annotation key for propagating span context to Tekton for distributed tracing.
+	SpanContextAnnotation = "tekton.dev/pipelinerunSpanContext"
 )
 
 var ParamsRe = regexp.MustCompile(`{{([^}]{2,})}}`)

pkg/kubeinteraction/labels.go

@@ -1,6 +1,8 @@
 package kubeinteraction
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"strconv"
 
@@ -12,6 +14,9 @@ import (
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
 	"github.com/openshift-pipelines/pipelines-as-code/pkg/params/versiondata"
 	tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	"knative.dev/pkg/logging"
 )
 
 const (
@@ -21,11 +26,15 @@ const (
 	StateFailed    = "failed"
 )
 
-func AddLabelsAndAnnotations(event *info.Event, pipelineRun *tektonv1.PipelineRun, repo *apipac.Repository, providerConfig *info.ProviderConfig, paramsRun *params.Run) error {
+func AddLabelsAndAnnotations(ctx context.Context, event *info.Event, pipelineRun *tektonv1.PipelineRun, repo *apipac.Repository, providerConfig *info.ProviderConfig, paramsRun *params.Run) error {
 	if event == nil {
 		return fmt.Errorf("event should not be nil")
 	}
 	paramsinfo := paramsRun.Info
+
+	// Inject span context for distributed tracing
+	carrier := propagation.MapCarrier{}
+	otel.GetTextMapPropagator().Inject(ctx, carrier)
 	// Add labels on the soon-to-be created pipelinerun so UI/CLI can easily
 	// query them.
 	labels := map[string]string{
@@ -60,6 +69,18 @@ func AddLabelsAndAnnotations(event *info.Event, pipelineRun *tektonv1.PipelineRu
 			paramsinfo.Controller.Name, paramsinfo.Controller.Configmap, paramsinfo.Controller.Secret, paramsinfo.Controller.GlobalRepository),
 	}
 
+	// Add span context for distributed tracing if available
+	if len(carrier) > 0 {
+		if jsonBytes, err := json.Marshal(carrier); err != nil {
+			logging.FromContext(ctx).Errorf("failed to marshal span context carrier: %v", err)
+		} else {
+			if existing := pipelineRun.GetAnnotations()[keys.SpanContextAnnotation]; existing != "" {
+				logging.FromContext(ctx).Warnf("overwriting pre-existing %s annotation on PipelineRun template; honoring initiating event trace context", keys.SpanContextAnnotation)
+			}
+			annotations[keys.SpanContextAnnotation] = string(jsonBytes)
+		}
+	}
+
 	if event.PullRequestNumber != 0 {
 		labels[keys.PullRequest] = strconv.Itoa(event.PullRequestNumber)
 		annotations[keys.PullRequest] = strconv.Itoa(event.PullRequestNumber)

pkg/kubeinteraction/labels_test.go

@@ -1,6 +1,7 @@
 package kubeinteraction
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
@@ -69,7 +70,7 @@ func TestAddLabelsAndAnnotations(t *testing.T) {
 					Controller: tt.args.controllerInfo,
 				},
 			}
-			err := AddLabelsAndAnnotations(tt.args.event, tt.args.pipelineRun, tt.args.repo, &info.ProviderConfig{}, paramsRun)
+			err := AddLabelsAndAnnotations(context.Background(), tt.args.event, tt.args.pipelineRun, tt.args.repo, &info.ProviderConfig{}, paramsRun)
 			assert.NilError(t, err)
 			assert.Equal(t, tt.args.pipelineRun.Labels[keys.URLOrg], tt.args.event.Organization, "'%s' != %s",
 				tt.args.pipelineRun.Labels[keys.URLOrg], tt.args.event.Organization)

pkg/pipelineascode/pipelineascode.go

@@ -216,7 +216,7 @@ func (p *PacRun) startPR(ctx context.Context, match matcher.Match) (*tektonv1.Pi
 	)
 
 	// Add labels and annotations to pipelinerun
-	err := kubeinteraction.AddLabelsAndAnnotations(p.event, match.PipelineRun, match.Repo, p.vcx.GetConfig(), p.run)
+	err := kubeinteraction.AddLabelsAndAnnotations(ctx, p.event, match.PipelineRun, match.Repo, p.vcx.GetConfig(), p.run)
 	if err != nil {
 		p.logger.Errorf("Error adding labels/annotations to PipelineRun '%s' in namespace '%s': %v", match.PipelineRun.GetName(), match.Repo.GetNamespace(), err)
 	} else {