Skip to content

Commit b598c2a

Browse files
committed
Dedup NodeOperationDetail across all traces in window
Move dedup map from per-trace to window level so the same relationship from different traces produces only one document. Signed-off-by: Vamsi Manohar <reddyvam@amazon.com>
1 parent 217ad0c commit b598c2a

File tree

1 file changed

+20
-24
lines changed
  • data-prepper-plugins/otel-apm-service-map-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otel_apm_service_map

1 file changed

+20
-24
lines changed

data-prepper-plugins/otel-apm-service-map-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/otel_apm_service_map/OTelApmServiceMapProcessor.java

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,7 @@ private Collection<Record<Event>> processCurrentWindowSpans() {
416416

417417
final Map<MetricKey, MetricAggregationState> sumStateByKey = new HashMap<>();
418418
final Map<MetricKey, MetricAggregationState> histogramStateByKey = new HashMap<>();
419+
final Map<String, NodeOperationDetail> dedupedNodeDetails = new HashMap<>();
419420

420421
final Map<String, Collection<SpanStateData>> previousSpansByTraceId = buildSpansByTraceIdMap(previousWindow);
421422
final Map<String, Collection<SpanStateData>> currentSpansByTraceId = buildSpansByTraceIdMap(currentWindow);
@@ -428,10 +429,23 @@ private Collection<Record<Event>> processCurrentWindowSpans() {
428429
if (!traceData.getProcessingSpans().isEmpty()) {
429430
decorateSpansInTraceWithEphemeralStorage(traceData);
430431

431-
apmEvents.addAll(generateNodeOperationDetailEvents(traceData, currentTime, sumStateByKey, histogramStateByKey));
432+
generateNodeOperationDetailEvents(traceData, currentTime, sumStateByKey, histogramStateByKey, dedupedNodeDetails);
432433
}
433434
}
434435

436+
// Convert deduped NodeOperationDetails to events
437+
for (NodeOperationDetail detail : dedupedNodeDetails.values()) {
438+
final EventMetadata eventMetadata = new DefaultEventMetadata.Builder()
439+
.withEventType(EVENT_TYPE_OTEL_APM_SERVICE_MAP).build();
440+
441+
final Event event = eventFactory.eventBuilder(EventBuilder.class)
442+
.withEventMetadata(eventMetadata)
443+
.withData(detail)
444+
.build();
445+
446+
apmEvents.add(new Record<>(event));
447+
}
448+
435449
final List<JacksonMetric> metrics = ApmServiceMapMetricsUtil.createMetricsFromAggregatedState(sumStateByKey, histogramStateByKey);
436450
metrics.sort(Comparator.comparing(JacksonMetric::getTime));
437451

@@ -782,13 +796,11 @@ private void decorateServerSpansSecondPassWithEphemeralStorage(final ThreeWindow
782796
* @param metricsStateByKey Shared map for metric aggregation across all traces
783797
* @return Collection of NodeOperationDetail events
784798
*/
785-
private Collection<Record<Event>> generateNodeOperationDetailEvents(final ThreeWindowTraceDataWithDecorations traceData,
786-
final Instant currentTime,
787-
final Map<MetricKey, MetricAggregationState> sumStateByKey,
788-
final Map<MetricKey, MetricAggregationState> histogramStateByKey) {
789-
// Dedup NodeOperationDetail by hash — first occurrence wins
790-
final Map<String, NodeOperationDetail> dedupedNodeDetails = new HashMap<>();
791-
799+
private void generateNodeOperationDetailEvents(final ThreeWindowTraceDataWithDecorations traceData,
800+
final Instant currentTime,
801+
final Map<MetricKey, MetricAggregationState> sumStateByKey,
802+
final Map<MetricKey, MetricAggregationState> histogramStateByKey,
803+
final Map<String, NodeOperationDetail> dedupedNodeDetails) {
792804
// Step 1: CLIENT spans — primary emission path
793805
for (SpanStateData clientSpan : traceData.getProcessingSpans()) {
794806
if (SPAN_KIND_CLIENT.equals(clientSpan.getSpanKind())) {
@@ -859,22 +871,6 @@ private Collection<Record<Event>> generateNodeOperationDetailEvents(final ThreeW
859871
}
860872
}
861873
}
862-
863-
// Convert deduped NodeOperationDetails to events
864-
final Collection<Record<Event>> events = new HashSet<>();
865-
for (NodeOperationDetail detail : dedupedNodeDetails.values()) {
866-
final EventMetadata eventMetadata = new DefaultEventMetadata.Builder()
867-
.withEventType(EVENT_TYPE_OTEL_APM_SERVICE_MAP).build();
868-
869-
final Event event = eventFactory.eventBuilder(EventBuilder.class)
870-
.withEventMetadata(eventMetadata)
871-
.withData(detail)
872-
.build();
873-
874-
events.add(new Record<>(event));
875-
}
876-
877-
return events;
878874
}
879875

880876
/**

0 commit comments

Comments
 (0)