Skip to content

Commit 2b83cec

Browse files
ci-operatorclaude
andcommitted
feat: add distributed tracing for webhook handling and PipelineRun timing
Emit a PipelinesAsCode:ProcessEvent span covering the full webhook event lifecycle. Emit waitDuration and executeDuration timing spans for completed PipelineRuns. Propagate trace context onto created PipelineRuns via the tekton.dev/pipelinerunSpanContext annotation. Configure the Knative observability framework to read tracing config from the pipelines-as-code-config-observability ConfigMap. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent baaf5e1 commit 2b83cec

File tree

11 files changed

+705
-7
lines changed

11 files changed

+705
-7
lines changed

cmd/pipelines-as-code-controller/main.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ func main() {
4040
loggerConfigurator := evadapter.NewLoggerConfiguratorFromConfigMap(PACControllerLogKey, loggerConfiguratorOpt)
4141
copts := []evadapter.ConfiguratorOption{
4242
evadapter.WithLoggerConfigurator(loggerConfigurator),
43-
evadapter.WithObservabilityConfigurator(evadapter.NewObservabilityConfiguratorFromConfigMap()),
43+
evadapter.WithObservabilityConfigurator(evadapter.NewObservabilityConfiguratorFromConfigMap(
44+
evadapter.WithObservabilityConfiguratorConfigMapName("pipelines-as-code-config-observability"),
45+
)),
4446
evadapter.WithCloudEventsStatusReporterConfigurator(evadapter.NewCloudEventsReporterConfiguratorFromConfigMap()),
4547
}
4648
// put logger configurator to ctx

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ require (
3131
gitlab.com/gitlab-org/api/client-go v1.14.0
3232
go.opentelemetry.io/otel v1.39.0
3333
go.opentelemetry.io/otel/metric v1.39.0
34+
go.opentelemetry.io/otel/sdk v1.39.0
3435
go.opentelemetry.io/otel/sdk/metric v1.39.0
36+
go.opentelemetry.io/otel/trace v1.39.0
3537
go.uber.org/zap v1.27.1
3638
golang.org/x/exp v0.0.0-20260112195511-716be5621a96
3739
golang.org/x/oauth2 v0.35.0
@@ -93,8 +95,6 @@ require (
9395
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 // indirect
9496
go.opentelemetry.io/otel/exporters/prometheus v0.61.0 // indirect
9597
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0 // indirect
96-
go.opentelemetry.io/otel/sdk v1.39.0 // indirect
97-
go.opentelemetry.io/otel/trace v1.39.0 // indirect
9898
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
9999
go.uber.org/atomic v1.11.0 // indirect
100100
go.yaml.in/yaml/v2 v2.4.3 // indirect

pkg/adapter/adapter.go

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ import (
2323
"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/gitea"
2424
"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/github"
2525
"github.com/openshift-pipelines/pipelines-as-code/pkg/provider/gitlab"
26+
"github.com/openshift-pipelines/pipelines-as-code/pkg/tracing"
27+
"go.opentelemetry.io/otel"
28+
"go.opentelemetry.io/otel/propagation"
29+
"go.opentelemetry.io/otel/trace"
2630
"go.uber.org/zap"
2731
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2832
"knative.dev/eventing/pkg/adapter/v2"
@@ -192,6 +196,26 @@ func (l listener) handleEvent(ctx context.Context) http.HandlerFunc {
192196
}
193197
gitProvider.SetPacInfo(&pacInfo)
194198

199+
// Extract inbound trace context from request headers for distributed tracing
200+
tracedCtx := otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(request.Header))
201+
202+
// Start a span for webhook handling
203+
tracer := otel.Tracer(tracing.TracerName)
204+
tracedCtx, span := tracer.Start(tracedCtx, "PipelinesAsCode:ProcessEvent",
205+
trace.WithSpanKind(trace.SpanKindServer),
206+
)
207+
208+
span.SetAttributes(
209+
tracing.VCSEventTypeKey.String(l.event.EventType),
210+
tracing.VCSProviderKey.String(gitProvider.GetConfig().Name),
211+
)
212+
if l.event.URL != "" {
213+
span.SetAttributes(tracing.VCSRepositoryKey.String(l.event.URL))
214+
}
215+
if l.event.SHA != "" {
216+
span.SetAttributes(tracing.VCSRevisionKey.String(l.event.SHA))
217+
}
218+
195219
s := sinker{
196220
run: l.run,
197221
vcx: gitProvider,
@@ -207,8 +231,10 @@ func (l listener) handleEvent(ctx context.Context) http.HandlerFunc {
207231
localRequest := request.Clone(request.Context())
208232

209233
go func() {
210-
err := s.processEvent(ctx, localRequest)
234+
defer span.End()
235+
err := s.processEvent(tracedCtx, localRequest)
211236
if err != nil {
237+
span.RecordError(err)
212238
logger.Errorf("an error occurred: %v", err)
213239
}
214240
}()

pkg/apis/pipelinesascode/keys/keys.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ const (
6666
GithubApplicationID = "github-application-id"
6767
GithubPrivateKey = "github-private-key"
6868
ResultsRecordSummary = "results.tekton.dev/recordSummaryAnnotations"
69+
70+
// SpanContextAnnotation is the annotation key for propagating span context to Tekton for distributed tracing
71+
SpanContextAnnotation = "tekton.dev/pipelinerunSpanContext"
6972
)
7073

7174
var ParamsRe = regexp.MustCompile(`{{([^}]{2,})}}`)

pkg/kubeinteraction/labels.go

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package kubeinteraction
22

33
import (
4+
"context"
5+
"encoding/json"
46
"fmt"
57
"strconv"
68

@@ -12,6 +14,9 @@ import (
1214
"github.com/openshift-pipelines/pipelines-as-code/pkg/params/info"
1315
"github.com/openshift-pipelines/pipelines-as-code/pkg/params/versiondata"
1416
tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
17+
"go.opentelemetry.io/otel"
18+
"go.opentelemetry.io/otel/propagation"
19+
"knative.dev/pkg/logging"
1520
)
1621

1722
const (
@@ -21,11 +26,15 @@ const (
2126
StateFailed = "failed"
2227
)
2328

24-
func AddLabelsAndAnnotations(event *info.Event, pipelineRun *tektonv1.PipelineRun, repo *apipac.Repository, providerConfig *info.ProviderConfig, paramsRun *params.Run) error {
29+
func AddLabelsAndAnnotations(ctx context.Context, event *info.Event, pipelineRun *tektonv1.PipelineRun, repo *apipac.Repository, providerConfig *info.ProviderConfig, paramsRun *params.Run) error {
2530
if event == nil {
2631
return fmt.Errorf("event should not be nil")
2732
}
2833
paramsinfo := paramsRun.Info
34+
35+
// Inject span context for distributed tracing
36+
carrier := propagation.MapCarrier{}
37+
otel.GetTextMapPropagator().Inject(ctx, carrier)
2938
// Add labels on the soon-to-be created pipelinerun so UI/CLI can easily
3039
// query them.
3140
labels := map[string]string{
@@ -59,6 +68,16 @@ func AddLabelsAndAnnotations(event *info.Event, pipelineRun *tektonv1.PipelineRu
5968
paramsinfo.Controller.Name, paramsinfo.Controller.Configmap, paramsinfo.Controller.Secret, paramsinfo.Controller.GlobalRepository),
6069
}
6170

71+
// Add span context for distributed tracing if available
72+
if len(carrier) > 0 {
73+
if jsonBytes, err := json.Marshal(map[string]string(carrier)); err == nil {
74+
if existing := pipelineRun.GetAnnotations()[keys.SpanContextAnnotation]; existing != "" {
75+
logging.FromContext(ctx).Warnf("overwriting pre-existing %s annotation on PipelineRun template; honoring initiating event trace context", keys.SpanContextAnnotation)
76+
}
77+
annotations[keys.SpanContextAnnotation] = string(jsonBytes)
78+
}
79+
}
80+
6281
if event.PullRequestNumber != 0 {
6382
labels[keys.PullRequest] = strconv.Itoa(event.PullRequestNumber)
6483
annotations[keys.PullRequest] = strconv.Itoa(event.PullRequestNumber)

pkg/kubeinteraction/labels_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package kubeinteraction
22

33
import (
4+
"context"
45
"fmt"
56
"testing"
67

@@ -68,7 +69,7 @@ func TestAddLabelsAndAnnotations(t *testing.T) {
6869
Controller: tt.args.controllerInfo,
6970
},
7071
}
71-
err := AddLabelsAndAnnotations(tt.args.event, tt.args.pipelineRun, tt.args.repo, &info.ProviderConfig{}, paramsRun)
72+
err := AddLabelsAndAnnotations(context.Background(), tt.args.event, tt.args.pipelineRun, tt.args.repo, &info.ProviderConfig{}, paramsRun)
7273
assert.NilError(t, err)
7374
assert.Equal(t, tt.args.pipelineRun.Labels[keys.URLOrg], tt.args.event.Organization, "'%s' != %s",
7475
tt.args.pipelineRun.Labels[keys.URLOrg], tt.args.event.Organization)

pkg/pipelineascode/pipelineascode.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ func (p *PacRun) startPR(ctx context.Context, match matcher.Match) (*tektonv1.Pi
250250
}
251251

252252
// Add labels and annotations to pipelinerun
253-
err := kubeinteraction.AddLabelsAndAnnotations(p.event, match.PipelineRun, match.Repo, p.vcx.GetConfig(), p.run)
253+
err := kubeinteraction.AddLabelsAndAnnotations(ctx, p.event, match.PipelineRun, match.Repo, p.vcx.GetConfig(), p.run)
254254
if err != nil {
255255
p.logger.Errorf("Error adding labels/annotations to PipelineRun '%s' in namespace '%s': %v", match.PipelineRun.GetName(), match.Repo.GetNamespace(), err)
256256
} else {

pkg/reconciler/emit_traces.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package reconciler
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
7+
"github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/keys"
8+
"github.com/openshift-pipelines/pipelines-as-code/pkg/tracing"
9+
tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
10+
"go.opentelemetry.io/otel"
11+
"go.opentelemetry.io/otel/attribute"
12+
"go.opentelemetry.io/otel/propagation"
13+
semconv "go.opentelemetry.io/otel/semconv/v1.37.0"
14+
"go.opentelemetry.io/otel/trace"
15+
corev1 "k8s.io/api/core/v1"
16+
"knative.dev/pkg/apis"
17+
)
18+
19+
const (
20+
applicationLabel = "appstudio.openshift.io/application"
21+
componentLabel = "appstudio.openshift.io/component"
22+
stageBuild = "build"
23+
)
24+
25+
26+
// extractSpanContext extracts the trace context from the pipelinerunSpanContext annotation.
27+
func extractSpanContext(pr *tektonv1.PipelineRun) (context.Context, bool) {
28+
raw, ok := pr.GetAnnotations()[keys.SpanContextAnnotation]
29+
if !ok || raw == "" {
30+
return nil, false
31+
}
32+
var carrierMap map[string]string
33+
if err := json.Unmarshal([]byte(raw), &carrierMap); err != nil {
34+
return nil, false
35+
}
36+
carrier := propagation.MapCarrier(carrierMap)
37+
prop := propagation.TraceContext{}
38+
ctx := prop.Extract(context.Background(), carrier)
39+
sc := trace.SpanContextFromContext(ctx)
40+
if !sc.IsValid() {
41+
return nil, false
42+
}
43+
return ctx, true
44+
}
45+
46+
// emitTimingSpans emits wait_duration and execute_duration spans for a completed build PipelineRun.
47+
func emitTimingSpans(pr *tektonv1.PipelineRun) {
48+
parentCtx, ok := extractSpanContext(pr)
49+
if !ok {
50+
return
51+
}
52+
53+
tracer := otel.GetTracerProvider().Tracer(tracing.TracerName)
54+
commonAttrs := buildCommonAttributes(pr)
55+
56+
// Emit waitDuration: creationTimestamp -> status.startTime
57+
if pr.Status.StartTime != nil {
58+
_, waitSpan := tracer.Start(parentCtx, "waitDuration",
59+
trace.WithTimestamp(pr.CreationTimestamp.Time),
60+
trace.WithAttributes(commonAttrs...),
61+
)
62+
waitSpan.End(trace.WithTimestamp(pr.Status.StartTime.Time))
63+
}
64+
65+
// Emit executeDuration: status.startTime -> status.completionTime
66+
if pr.Status.StartTime != nil && pr.Status.CompletionTime != nil {
67+
execAttrs := append(commonAttrs, buildExecuteAttributes(pr)...)
68+
_, execSpan := tracer.Start(parentCtx, "executeDuration",
69+
trace.WithTimestamp(pr.Status.StartTime.Time),
70+
trace.WithAttributes(execAttrs...),
71+
)
72+
execSpan.End(trace.WithTimestamp(pr.Status.CompletionTime.Time))
73+
}
74+
}
75+
76+
// buildCommonAttributes returns span attributes common to both timing spans.
77+
func buildCommonAttributes(pr *tektonv1.PipelineRun) []attribute.KeyValue {
78+
attrs := []attribute.KeyValue{
79+
semconv.K8SNamespaceName(pr.GetNamespace()),
80+
tracing.TektonPipelineRunNameKey.String(pr.GetName()),
81+
tracing.TektonPipelineRunUIDKey.String(string(pr.GetUID())),
82+
tracing.DeliveryStageKey.String(stageBuild),
83+
tracing.DeliveryApplicationKey.String(pr.GetLabels()[applicationLabel]),
84+
}
85+
if component := pr.GetLabels()[componentLabel]; component != "" {
86+
attrs = append(attrs, tracing.DeliveryComponentKey.String(component))
87+
}
88+
return attrs
89+
}
90+
91+
// buildExecuteAttributes returns span attributes specific to execute_duration.
92+
func buildExecuteAttributes(pr *tektonv1.PipelineRun) []attribute.KeyValue {
93+
cond := pr.Status.GetCondition(apis.ConditionSucceeded)
94+
success := true
95+
reason := ""
96+
if cond != nil {
97+
reason = cond.Reason
98+
if cond.Status == corev1.ConditionFalse {
99+
success = false
100+
}
101+
}
102+
return []attribute.KeyValue{
103+
tracing.DeliverySuccessKey.Bool(success),
104+
tracing.DeliveryReasonKey.String(reason),
105+
}
106+
}

0 commit comments

Comments
 (0)