Extract trace context from http request and connect with replicas (#2603)

* Extract trace context from http request and connect with replicas

* Move trace extraction to common reusable utility

Author: bayarcher

client/clientservice/src/request_service.cpp

@@ -15,7 +15,9 @@
 #include "client/clientservice/request_service.hpp"
 #include "client/concordclient/concord_client.hpp"
 #include "concord_client_request.pb.h"
+#include "client/thin-replica-client/trace_contexts.hpp"
 
+using namespace client::thin_replica_client;
 using namespace vmware::concord::client::concord_client_request::v1;
 
 namespace concord::client::clientservice {
@@ -173,18 +175,21 @@ void RequestServiceCallData::sendToConcordClient() {
     this->populateResult(grpc::Status::OK);
   };
 
+  auto tracer = opentracing::Tracer::Global();
+  auto parent_span = TraceContexts::ExtractSpanFromMetadata(*tracer, ctx_);
+
   if (request_.read_only()) {
     bft::client::ReadConfig config;
     config.request = req_config;
-    auto span = opentracing::Tracer::Global()->StartSpan("send_ro", {});
+    auto span = opentracing::Tracer::Global()->StartSpan("send_ro", {opentracing::ChildOf(parent_span.get())});
     std::ostringstream carrier;
     opentracing::Tracer::Global()->Inject(span->context(), carrier);
     config.request.span_context = carrier.str();
     client_->send(config, std::move(msg), callback);
   } else {
     bft::client::WriteConfig config;
     config.request = req_config;
-    auto span = opentracing::Tracer::Global()->StartSpan("send", {});
+    auto span = opentracing::Tracer::Global()->StartSpan("send", {opentracing::ChildOf(parent_span.get())});
     std::ostringstream carrier;
     opentracing::Tracer::Global()->Inject(span->context(), carrier);
     config.request.span_context = carrier.str();

client/thin-replica-client/include/client/thin-replica-client/trace_contexts.hpp

@@ -15,6 +15,7 @@
 #define THIN_REPLICA_CLIENT_TRACE_CONTEXTS_HPP_
 
 #include <opentracing/span.h>
+#include <grpcpp/server.h>
 #include <optional>
 
 #include "thin_replica.pb.h"
@@ -30,12 +31,14 @@ namespace client::thin_replica_client {
 class TraceContexts {
  public:
   using SpanPtr = std::unique_ptr<opentracing::Span>;
+  using SpanCtxPtr = std::unique_ptr<opentracing::SpanContext>;
 
   static void InjectSpan(const SpanPtr& span, cc::EventVariant& update);
   static expected<std::unique_ptr<opentracing::SpanContext>> ExtractSpan(const cc::EventVariant& update);
   static SpanPtr CreateChildSpanFromBinary(const std::string& trace_context,
                                            const std::string& child_name,
                                            const std::string& correlation_id = {});
+  static SpanCtxPtr ExtractSpanFromMetadata(const opentracing::Tracer& tracer, const grpc::ServerContext& context);
 };
 
 }  // namespace client::thin_replica_client

client/thin-replica-client/src/trace_contexts.cpp

@@ -111,4 +111,47 @@ SpanPtr TraceContexts::CreateChildSpanFromBinary(const std::string& trace_contex
   return opentracing::Tracer::Global()->StartSpan(
       child_name, {opentracing::FollowsFrom(&**parent_span_context), opentracing::SetTag{kCorrelationIdTag, cid}});
 }
+
+struct GrpcMetadataCarrierForReading : opentracing::TextMapReader {
+  GrpcMetadataCarrierForReading(const std::multimap<grpc::string_ref, grpc::string_ref>& client_metadata)
+      : client_metadata_(client_metadata) {}
+
+  using F = std::function<opentracing::expected<void>(opentracing::string_view, opentracing::string_view)>;
+
+  opentracing::expected<void> ForeachKey(F f) const override {
+    // Iterate through all key-value pairs, the tracer will use the relevant keys
+    // to extract a span context.
+    for (auto& key_value : client_metadata_) {
+      auto was_successful = f(FromStringRef(key_value.first), FromStringRef(key_value.second));
+      if (!was_successful) {
+        // If the callback returns and unexpected value, bail out of the loop.
+        return was_successful;
+      }
+    }
+    // Indicate successful iteration.
+    return {};
+  }
+
+  opentracing::expected<opentracing::string_view> LookupKey(opentracing::string_view key) const override {
+    auto find_it = client_metadata_.find(grpc::string_ref(key));
+    if (find_it != client_metadata_.end()) {
+      return opentracing::make_unexpected(opentracing::key_not_found_error);
+    }
+    return opentracing::string_view{FromStringRef(find_it->second)};
+  }
+
+ private:
+  static std::string FromStringRef(const grpc::string_ref& string_ref) {
+    return std::string(string_ref.data(), string_ref.size());
+  }
+
+  const std::multimap<grpc::string_ref, grpc::string_ref>& client_metadata_;
+};
+
+TraceContexts::SpanCtxPtr TraceContexts::ExtractSpanFromMetadata(const opentracing::Tracer& tracer,
+                                                                 const grpc::ServerContext& context) {
+  GrpcMetadataCarrierForReading metadata_carrier(context.client_metadata());
+  return *tracer.Extract(metadata_carrier);
+}
+
 }  // namespace client::thin_replica_client