From 938d19f63edf356013febe5de53eba8326d2e2dc Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 7 Sep 2023 14:07:57 -0700 Subject: [PATCH] [GSM Observability] Add mesh_id support in injected labels (#34247) Changes - 1) Change local mesh labels to not be reported on 'started' metrics at all (even those that we know about) to be consistent. (Since xDS labels atleast on the server side would not be available on started metric.) 2) Add mesh_id as a local label that is populated by reading the xDS bootstrap. As part of this, also added a minimal xds bootstrap parsing logic. --- src/cpp/ext/gsm/BUILD | 5 + src/cpp/ext/gsm/metadata_exchange.cc | 163 ++++++++++++----- src/cpp/ext/gsm/metadata_exchange.h | 11 +- src/cpp/ext/otel/key_value_iterable.h | 31 +--- src/cpp/ext/otel/otel_call_tracer.h | 3 +- src/cpp/ext/otel/otel_client_filter.cc | 19 +- src/cpp/ext/otel/otel_plugin.h | 11 +- src/cpp/ext/otel/otel_server_call_tracer.cc | 23 +-- test/cpp/ext/gsm/BUILD | 16 ++ test/cpp/ext/gsm/mesh_id_test.cc | 137 +++++++++++++++ test/cpp/ext/gsm/metadata_exchange_test.cc | 184 ++++++++++++++------ 11 files changed, 446 insertions(+), 157 deletions(-) create mode 100644 test/cpp/ext/gsm/mesh_id_test.cc diff --git a/src/cpp/ext/gsm/BUILD b/src/cpp/ext/gsm/BUILD index 33f07f3b48c..b7524dd3afa 100644 --- a/src/cpp/ext/gsm/BUILD +++ b/src/cpp/ext/gsm/BUILD @@ -57,6 +57,11 @@ grpc_cc_library( "//:grpc_base", "//:protobuf_struct_upb", "//src/core:env", + "//src/core:error", + "//src/core:json", + "//src/core:json_args", + "//src/core:json_object_loader", + "//src/core:json_reader", "//src/core:slice", "//src/cpp/ext/otel:otel_plugin", ], diff --git a/src/cpp/ext/gsm/metadata_exchange.cc b/src/cpp/ext/gsm/metadata_exchange.cc index 890b3ae5e7f..89f79730c2c 100644 --- a/src/cpp/ext/gsm/metadata_exchange.cc +++ b/src/cpp/ext/gsm/metadata_exchange.cc @@ -22,13 +22,17 @@ #include +#include #include #include #include #include "absl/meta/type_traits.h" +#include "absl/status/statusor.h" #include "absl/strings/escaping.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" #include "absl/types/optional.h" #include "absl/types/variant.h" #include "google/protobuf/struct.upb.h" @@ -37,7 +41,16 @@ #include "upb/mem/arena.h" #include "upb/upb.hpp" +#include + #include "src/core/lib/gprpp/env.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_args.h" +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/json/json_reader.h" +#include "src/core/lib/slice/slice_internal.h" namespace grpc { namespace internal { @@ -56,6 +69,8 @@ constexpr absl::string_view kMetadataExchangeLocationKey = "location"; constexpr absl::string_view kMetadataExchangeProjectIdKey = "project_id"; constexpr absl::string_view kMetadataExchangeCanonicalServiceKey = "canonical_service"; +// The keys that will be used for the local attributes when recording metrics. +constexpr absl::string_view kMeshIdAttribute = "gsm.mesh_id"; // The keys that will be used for the peer attributes when recording metrics. constexpr absl::string_view kPeerTypeAttribute = "gsm.remote_workload_type"; constexpr absl::string_view kPeerPodNameAttribute = @@ -77,6 +92,62 @@ constexpr absl::string_view kGkeType = "gcp_kubernetes_engine"; enum class GcpResourceType : std::uint8_t { kGke, kUnknown }; +// A minimal class for helping with the information we need from the xDS +// bootstrap file for GSM Observability reasons. +class XdsBootstrapForGSM { + public: + class Node { + public: + const std::string& id() const { return id_; } + + static const grpc_core::JsonLoaderInterface* JsonLoader( + const grpc_core::JsonArgs&) { + static const auto* loader = + grpc_core::JsonObjectLoader().Field("id", &Node::id_).Finish(); + return loader; + } + + private: + std::string id_; + }; + + const Node& node() const { return node_; } + + static const grpc_core::JsonLoaderInterface* JsonLoader( + const grpc_core::JsonArgs&) { + static const auto* loader = + grpc_core::JsonObjectLoader() + .Field("node", &XdsBootstrapForGSM::node_) + .Finish(); + return loader; + } + + private: + Node node_; +}; + +// Returns an empty string if no bootstrap config is found. +std::string GetXdsBootstrapContents() { + // First, try GRPC_XDS_BOOTSTRAP env var. + auto path = grpc_core::GetEnv("GRPC_XDS_BOOTSTRAP"); + if (path.has_value()) { + grpc_slice contents; + grpc_error_handle error = + grpc_load_file(path->c_str(), /*add_null_terminator=*/true, &contents); + if (!error.ok()) return ""; + std::string contents_str(grpc_core::StringViewFromSlice(contents)); + grpc_core::CSliceUnref(contents); + return contents_str; + } + // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var. + auto env_config = grpc_core::GetEnv("GRPC_XDS_BOOTSTRAP_CONFIG"); + if (env_config.has_value()) { + return std::move(*env_config); + } + // No bootstrap config found. + return ""; +} + GcpResourceType StringToGcpResourceType(absl::string_view type) { if (type == kGkeType) { return GcpResourceType::kGke; @@ -133,41 +204,25 @@ absl::string_view GetStringValueFromUpbStruct(google_protobuf_Struct* struct_pb, return "unknown"; } -class LocalLabelsIterable : public LabelsIterable { +class MeshLabelsIterable : public LabelsIterable { public: - explicit LocalLabelsIterable( - const std::vector>& labels) - : labels_(labels) {} - - absl::optional> Next() - override { - if (pos_ >= labels_.size()) { - return absl::nullopt; - } - return labels_[pos_++]; - } - - size_t Size() const override { return labels_.size(); } - - void ResetIteratorPosition() override { pos_ = 0; } - - private: - size_t pos_ = 0; - const std::vector>& labels_; -}; - -class PeerLabelsIterable : public LabelsIterable { - public: - explicit PeerLabelsIterable(grpc_core::Slice remote_metadata) - : metadata_(std::move(remote_metadata)) {} + explicit MeshLabelsIterable( + const std::vector>& + local_labels, + grpc_core::Slice remote_metadata) + : local_labels_(local_labels), metadata_(std::move(remote_metadata)) {} absl::optional> Next() override { auto& struct_pb = GetDecodedMetadata(); + size_t local_labels_size = local_labels_.size(); + if (pos_ < local_labels_size) { + return local_labels_[pos_++]; + } if (struct_pb.struct_pb == nullptr) { return absl::nullopt; } - if (++pos_ == 1) { + if (++pos_ == local_labels_size + 1) { return std::make_pair(kPeerTypeAttribute, GetStringValueFromUpbStruct( struct_pb.struct_pb, kMetadataExchangeTypeKey, @@ -176,14 +231,15 @@ class PeerLabelsIterable : public LabelsIterable { // Only handle GKE type for now. switch (type_) { case GcpResourceType::kGke: - if (pos_ - 2 >= kGkeAttributeList.size()) { + if ((pos_ - 2 - local_labels_size) >= kGkeAttributeList.size()) { return absl::nullopt; } return std::make_pair( - kGkeAttributeList[pos_ - 2].otel_attribute, + kGkeAttributeList[pos_ - 2 - local_labels_size].otel_attribute, GetStringValueFromUpbStruct( struct_pb.struct_pb, - kGkeAttributeList[pos_ - 2].metadata_attribute, + kGkeAttributeList[pos_ - 2 - local_labels_size] + .metadata_attribute, struct_pb.arena.ptr())); case GcpResourceType::kUnknown: return absl::nullopt; @@ -193,12 +249,12 @@ class PeerLabelsIterable : public LabelsIterable { size_t Size() const override { auto& struct_pb = GetDecodedMetadata(); if (struct_pb.struct_pb == nullptr) { - return 0; + return local_labels_.size(); } if (type_ != GcpResourceType::kGke) { - return 1; + return local_labels_.size() + 1; } - return kGkeAttributeList.size(); + return local_labels_.size() + kGkeAttributeList.size() + 1; } void ResetIteratorPosition() override { pos_ = 0; } @@ -248,17 +304,43 @@ class PeerLabelsIterable : public LabelsIterable { return struct_pb; } + const std::vector>& local_labels_; // Holds either the metadata slice or the decoded proto struct. mutable absl::variant metadata_; mutable GcpResourceType type_; uint32_t pos_ = 0; }; -constexpr std::array - PeerLabelsIterable::kGkeAttributeList; +constexpr std::array + MeshLabelsIterable::kGkeAttributeList; } // namespace +// Returns the mesh ID by reading and parsing the bootstrap file. Returns +// "unknown" if for some reason, mesh ID could not be figured out. +std::string GetMeshId() { + auto json = grpc_core::JsonParse(GetXdsBootstrapContents()); + if (!json.ok()) { + return "unknown"; + } + auto bootstrap = grpc_core::LoadFromJson(*json); + if (!bootstrap.ok()) { + return "unknown"; + } + // The format of the Node ID is - + // projects/[GCP Project number]/networks/mesh:[Mesh ID]/nodes/[UUID] + std::vector parts = + absl::StrSplit(bootstrap->node().id(), '/'); + if (parts.size() != 6) { + return "unknown"; + } + absl::string_view mesh_id = parts[3]; + if (!absl::ConsumePrefix(&mesh_id, "mesh:")) { + return "unknown"; + } + return std::string(mesh_id); +} + ServiceMeshLabelsInjector::ServiceMeshLabelsInjector( const opentelemetry::sdk::common::AttributeMap& map) { upb::Arena arena; @@ -317,21 +399,18 @@ ServiceMeshLabelsInjector::ServiceMeshLabelsInjector( absl::Base64Escape(absl::string_view(output, output_length))); // Fill up local labels map. The rest we get from the detected Resource and // from the peer. - // TODO(yashykt): Add mesh_id + local_labels_.emplace_back(kMeshIdAttribute, GetMeshId()); } -std::unique_ptr ServiceMeshLabelsInjector::GetPeerLabels( +std::unique_ptr ServiceMeshLabelsInjector::GetLabels( grpc_metadata_batch* incoming_initial_metadata) { auto peer_metadata = incoming_initial_metadata->Take(grpc_core::XEnvoyPeerMetadata()); if (!peer_metadata.has_value()) { return nullptr; } - return std::make_unique(*std::move(peer_metadata)); -} - -std::unique_ptr ServiceMeshLabelsInjector::GetLocalLabels() { - return std::make_unique(local_labels_); + return std::make_unique(local_labels_, + *std::move(peer_metadata)); } void ServiceMeshLabelsInjector::AddLabels( diff --git a/src/cpp/ext/gsm/metadata_exchange.h b/src/cpp/ext/gsm/metadata_exchange.h index fc19d151ae2..91368894357 100644 --- a/src/cpp/ext/gsm/metadata_exchange.h +++ b/src/cpp/ext/gsm/metadata_exchange.h @@ -42,13 +42,9 @@ class ServiceMeshLabelsInjector : public LabelsInjector { const opentelemetry::sdk::common::AttributeMap& map); // Read the incoming initial metadata to get the set of labels to be added to // metrics. - std::unique_ptr GetPeerLabels( + std::unique_ptr GetLabels( grpc_metadata_batch* incoming_initial_metadata) override; - // Get the local labels to be added to metrics. To be used when the peer - // metadata is not available, for example, for started RPCs metric. - std::unique_ptr GetLocalLabels() override; - // Modify the outgoing initial metadata with metadata information to be sent // to the peer. void AddLabels(grpc_metadata_batch* outgoing_initial_metadata) override; @@ -58,6 +54,11 @@ class ServiceMeshLabelsInjector : public LabelsInjector { grpc_core::Slice serialized_labels_to_send_; }; +// Returns the mesh ID by reading and parsing the bootstrap file. Returns +// "unknown" if for some reason, mesh ID could not be figured out. +// EXPOSED FOR TESTING PURPOSES ONLY. +std::string GetMeshId(); + } // namespace internal } // namespace grpc diff --git a/src/cpp/ext/otel/key_value_iterable.h b/src/cpp/ext/otel/key_value_iterable.h index a76c6c9edcd..f88f3dbac7c 100644 --- a/src/cpp/ext/otel/key_value_iterable.h +++ b/src/cpp/ext/otel/key_value_iterable.h @@ -49,30 +49,19 @@ inline opentelemetry::nostd::string_view AbslStrViewToOTelStrView( class KeyValueIterable : public opentelemetry::common::KeyValueIterable { public: explicit KeyValueIterable( - LabelsIterable* local_labels_iterable, - LabelsIterable* peer_labels_iterable, + LabelsIterable* injected_labels_iterable, absl::Span> additional_labels) - : local_labels_iterable_(local_labels_iterable), - peer_labels_iterable_(peer_labels_iterable), + : injected_labels_iterable_(injected_labels_iterable), additional_labels_(additional_labels) {} bool ForEachKeyValue(opentelemetry::nostd::function_ref< bool(opentelemetry::nostd::string_view, opentelemetry::common::AttributeValue)> callback) const noexcept override { - if (local_labels_iterable_ != nullptr) { - local_labels_iterable_->ResetIteratorPosition(); - while (const auto& pair = local_labels_iterable_->Next()) { - if (!callback(AbslStrViewToOTelStrView(pair->first), - AbslStrViewToOTelStrView(pair->second))) { - return false; - } - } - } - if (peer_labels_iterable_ != nullptr) { - peer_labels_iterable_->ResetIteratorPosition(); - while (const auto& pair = peer_labels_iterable_->Next()) { + if (injected_labels_iterable_ != nullptr) { + injected_labels_iterable_->ResetIteratorPosition(); + while (const auto& pair = injected_labels_iterable_->Next()) { if (!callback(AbslStrViewToOTelStrView(pair->first), AbslStrViewToOTelStrView(pair->second))) { return false; @@ -89,16 +78,14 @@ class KeyValueIterable : public opentelemetry::common::KeyValueIterable { } size_t size() const noexcept override { - return (local_labels_iterable_ != nullptr ? local_labels_iterable_->Size() - : 0) + - (peer_labels_iterable_ != nullptr ? peer_labels_iterable_->Size() - : 0) + + return (injected_labels_iterable_ != nullptr + ? injected_labels_iterable_->Size() + : 0) + additional_labels_.size(); } private: - LabelsIterable* local_labels_iterable_; - LabelsIterable* peer_labels_iterable_; + LabelsIterable* injected_labels_iterable_; absl::Span> additional_labels_; }; diff --git a/src/cpp/ext/otel/otel_call_tracer.h b/src/cpp/ext/otel/otel_call_tracer.h index 37f5445c035..e4165b770c4 100644 --- a/src/cpp/ext/otel/otel_call_tracer.h +++ b/src/cpp/ext/otel/otel_call_tracer.h @@ -95,8 +95,7 @@ class OpenTelemetryCallTracer : public grpc_core::ClientCallTracer { const bool arena_allocated_; // Start time (for measuring latency). absl::Time start_time_; - std::unique_ptr local_labels_; - std::unique_ptr peer_labels_; + std::unique_ptr injected_labels_; }; explicit OpenTelemetryCallTracer(OpenTelemetryClientFilter* parent, diff --git a/src/cpp/ext/otel/otel_client_filter.cc b/src/cpp/ext/otel/otel_client_filter.cc index 13a6fce399c..646787c5b6c 100644 --- a/src/cpp/ext/otel/otel_client_filter.cc +++ b/src/cpp/ext/otel/otel_client_filter.cc @@ -104,27 +104,25 @@ OpenTelemetryCallTracer::OpenTelemetryCallAttemptTracer:: : parent_(parent), arena_allocated_(arena_allocated), start_time_(absl::Now()) { - // We don't have the peer labels at this point. - if (OTelPluginState().labels_injector != nullptr) { - local_labels_ = OTelPluginState().labels_injector->GetLocalLabels(); - } if (OTelPluginState().client.attempt.started != nullptr) { std::array, 2> additional_labels = { {{OTelMethodKey(), absl::StripPrefix(parent_->path_.as_string_view(), "/")}, {OTelTargetKey(), parent_->parent_->target()}}}; - KeyValueIterable labels(local_labels_.get(), peer_labels_.get(), - additional_labels); - OTelPluginState().client.attempt.started->Add(1, labels); + // We might not have all the injected labels that we want at this point, so + // avoid recording a subset of injected labels here. + OTelPluginState().client.attempt.started->Add( + 1, KeyValueIterable(/*injected_labels_iterable=*/nullptr, + additional_labels)); } } void OpenTelemetryCallTracer::OpenTelemetryCallAttemptTracer:: RecordReceivedInitialMetadata(grpc_metadata_batch* recv_initial_metadata) { if (OTelPluginState().labels_injector != nullptr) { - peer_labels_ = - OTelPluginState().labels_injector->GetPeerLabels(recv_initial_metadata); + injected_labels_ = + OTelPluginState().labels_injector->GetLabels(recv_initial_metadata); } } @@ -173,8 +171,7 @@ void OpenTelemetryCallTracer::OpenTelemetryCallAttemptTracer:: {OTelStatusKey(), grpc_status_code_to_string( static_cast(status.code()))}}}; - KeyValueIterable labels(local_labels_.get(), peer_labels_.get(), - additional_labels); + KeyValueIterable labels(injected_labels_.get(), additional_labels); if (OTelPluginState().client.attempt.duration != nullptr) { OTelPluginState().client.attempt.duration->Record( absl::ToDoubleSeconds(absl::Now() - start_time_), labels, diff --git a/src/cpp/ext/otel/otel_plugin.h b/src/cpp/ext/otel/otel_plugin.h index 36344563f0c..7cae5f83e62 100644 --- a/src/cpp/ext/otel/otel_plugin.h +++ b/src/cpp/ext/otel/otel_plugin.h @@ -63,17 +63,10 @@ class LabelsInjector { public: virtual ~LabelsInjector() {} // Read the incoming initial metadata to get the set of labels to be added to - // metrics. (Does not include the local labels.) - virtual std::unique_ptr GetPeerLabels( + // metrics. + virtual std::unique_ptr GetLabels( grpc_metadata_batch* incoming_initial_metadata) = 0; - // Get the local labels to be added to metrics. To be used when the peer - // metadata is not available, for example, for started RPCs metric. - // It is the responsibility of the implementation to make sure that the - // backing store for the absl::string_view remains valid for the lifetime of - // gRPC. - virtual std::unique_ptr GetLocalLabels() = 0; - // Modify the outgoing initial metadata with metadata information to be sent // to the peer. virtual void AddLabels(grpc_metadata_batch* outgoing_initial_metadata) = 0; diff --git a/src/cpp/ext/otel/otel_server_call_tracer.cc b/src/cpp/ext/otel/otel_server_call_tracer.cc index 652c9bf6f01..e5ab55b7dfd 100644 --- a/src/cpp/ext/otel/otel_server_call_tracer.cc +++ b/src/cpp/ext/otel/otel_server_call_tracer.cc @@ -54,12 +54,7 @@ namespace { class OpenTelemetryServerCallTracer : public grpc_core::ServerCallTracer { public: - OpenTelemetryServerCallTracer() : start_time_(absl::Now()) { - // We don't have the peer labels at this point. - if (OTelPluginState().labels_injector != nullptr) { - local_labels_ = OTelPluginState().labels_injector->GetLocalLabels(); - } - } + OpenTelemetryServerCallTracer() : start_time_(absl::Now()) {} std::string TraceId() override { // Not implemented @@ -82,7 +77,7 @@ class OpenTelemetryServerCallTracer : public grpc_core::ServerCallTracer { grpc_metadata_batch* send_initial_metadata) override { // Only add labels to outgoing metadata if labels were received from peer. if (OTelPluginState().labels_injector != nullptr && - peer_labels_ != nullptr) { + injected_labels_ != nullptr) { OTelPluginState().labels_injector->AddLabels(send_initial_metadata); } } @@ -135,8 +130,7 @@ class OpenTelemetryServerCallTracer : public grpc_core::ServerCallTracer { absl::Time start_time_; absl::Duration elapsed_time_; grpc_core::Slice path_; - std::unique_ptr local_labels_; - std::unique_ptr peer_labels_; + std::unique_ptr injected_labels_; }; void OpenTelemetryServerCallTracer::RecordReceivedInitialMetadata( @@ -144,15 +138,17 @@ void OpenTelemetryServerCallTracer::RecordReceivedInitialMetadata( path_ = recv_initial_metadata->get_pointer(grpc_core::HttpPathMetadata())->Ref(); if (OTelPluginState().labels_injector != nullptr) { - peer_labels_ = - OTelPluginState().labels_injector->GetPeerLabels(recv_initial_metadata); + injected_labels_ = + OTelPluginState().labels_injector->GetLabels(recv_initial_metadata); } std::array, 1> additional_labels = { {{OTelMethodKey(), absl::StripPrefix(path_.as_string_view(), "/")}}}; if (OTelPluginState().server.call.started != nullptr) { + // We might not have all the injected labels that we want at this point, so + // avoid recording a subset of injected labels here. OTelPluginState().server.call.started->Add( - 1, KeyValueIterable(local_labels_.get(), peer_labels_.get(), + 1, KeyValueIterable(/*injected_labels_iterable=*/nullptr, additional_labels)); } } @@ -171,8 +167,7 @@ void OpenTelemetryServerCallTracer::RecordEnd( {{OTelMethodKey(), absl::StripPrefix(path_.as_string_view(), "/")}, {OTelStatusKey(), grpc_status_code_to_string(final_info->final_status)}}}; - KeyValueIterable labels(local_labels_.get(), peer_labels_.get(), - additional_labels); + KeyValueIterable labels(injected_labels_.get(), additional_labels); if (OTelPluginState().server.call.duration != nullptr) { OTelPluginState().server.call.duration->Record( absl::ToDoubleSeconds(elapsed_time_), labels, diff --git a/test/cpp/ext/gsm/BUILD b/test/cpp/ext/gsm/BUILD index 71cd12945e6..34de6bbc54c 100644 --- a/test/cpp/ext/gsm/BUILD +++ b/test/cpp/ext/gsm/BUILD @@ -61,3 +61,19 @@ grpc_cc_test( "//test/cpp/ext/otel:otel_test_library", ], ) + +grpc_cc_test( + name = "mesh_id_test", + srcs = [ + "mesh_id_test.cc", + ], + external_deps = [ + "gtest", + ], + language = "C++", + deps = [ + "//:grpc++", + "//src/cpp/ext/gsm:gsm_observability", + "//test/core/util:grpc_test_util", + ], +) diff --git a/test/cpp/ext/gsm/mesh_id_test.cc b/test/cpp/ext/gsm/mesh_id_test.cc new file mode 100644 index 00000000000..2933703a595 --- /dev/null +++ b/test/cpp/ext/gsm/mesh_id_test.cc @@ -0,0 +1,137 @@ +// +// +// Copyright 2023 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#include "gtest/gtest.h" + +#include + +#include "src/core/lib/gpr/tmpfile.h" +#include "src/core/lib/gprpp/env.h" +#include "src/cpp/ext/gsm/metadata_exchange.h" +#include "test/core/util/test_config.h" + +namespace grpc { +namespace testing { +namespace { + +class TestScenario { + public: + enum class XdsBootstrapSource : std::uint8_t { kFromFile, kFromConfig }; + + explicit TestScenario(XdsBootstrapSource bootstrap_source) + : bootstrap_source_(bootstrap_source) {} + + static std::string Name(const ::testing::TestParamInfo& info) { + switch (info.param.bootstrap_source_) { + case TestScenario::XdsBootstrapSource::kFromFile: + return "BootstrapFromFile"; + case TestScenario::XdsBootstrapSource::kFromConfig: + return "BootstrapFromConfig"; + } + } + + XdsBootstrapSource bootstrap_source() const { return bootstrap_source_; } + + private: + XdsBootstrapSource bootstrap_source_; +}; + +class MeshIdTest : public ::testing::TestWithParam { + protected: + void SetBootstrap(const char* bootstrap) { + switch (GetParam().bootstrap_source()) { + case TestScenario::XdsBootstrapSource::kFromFile: { + ASSERT_EQ(bootstrap_file_name_, nullptr); + FILE* bootstrap_file = + gpr_tmpfile("gcp_observability_config", &bootstrap_file_name_); + fputs(bootstrap, bootstrap_file); + fclose(bootstrap_file); + grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP", bootstrap_file_name_); + break; + } + case TestScenario::XdsBootstrapSource::kFromConfig: + grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP_CONFIG", bootstrap); + break; + } + } + + ~MeshIdTest() override { + grpc_core::UnsetEnv("GRPC_GCP_OBSERVABILITY_CONFIG"); + grpc_core::UnsetEnv("GRPC_XDS_BOOTSTRAP"); + if (bootstrap_file_name_ != nullptr) { + remove(bootstrap_file_name_); + gpr_free(bootstrap_file_name_); + } + } + + char* bootstrap_file_name_ = nullptr; +}; + +TEST_P(MeshIdTest, Empty) { + SetBootstrap(""); + EXPECT_EQ(grpc::internal::GetMeshId(), "unknown"); +} + +TEST_P(MeshIdTest, NothingSet) { + EXPECT_EQ(grpc::internal::GetMeshId(), "unknown"); +} + +TEST_P(MeshIdTest, BadJson) { + SetBootstrap("{"); + EXPECT_EQ(grpc::internal::GetMeshId(), "unknown"); +} + +TEST_P(MeshIdTest, UnexpectedMeshIdFormatType1) { + SetBootstrap( + "{\"node\": {\"id\": " + "\"abcdef\"}}"); + EXPECT_EQ(grpc::internal::GetMeshId(), "unknown"); +} + +TEST_P(MeshIdTest, UnexpectedMeshIdFormatType2) { + SetBootstrap( + "{\"node\": {\"id\": " + "\"projects/1234567890/networks/mesh-id/nodes/" + "01234567-89ab-4def-8123-456789abcdef\"}}"); + EXPECT_EQ(grpc::internal::GetMeshId(), "unknown"); +} + +TEST_P(MeshIdTest, Basic) { + SetBootstrap( + "{\"node\": {\"id\": " + "\"projects/1234567890/networks/mesh:mesh-id/nodes/" + "01234567-89ab-4def-8123-456789abcdef\"}}"); + EXPECT_EQ(grpc::internal::GetMeshId(), "mesh-id"); +} + +INSTANTIATE_TEST_SUITE_P( + MeshId, MeshIdTest, + ::testing::Values( + TestScenario(TestScenario::XdsBootstrapSource::kFromFile), + TestScenario(TestScenario::XdsBootstrapSource::kFromConfig)), + &TestScenario::Name); + +} // namespace +} // namespace testing +} // namespace grpc + +int main(int argc, char** argv) { + grpc::testing::TestEnvironment env(&argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/cpp/ext/gsm/metadata_exchange_test.cc b/test/cpp/ext/gsm/metadata_exchange_test.cc index b3dc86f7ce4..62781714134 100644 --- a/test/cpp/ext/gsm/metadata_exchange_test.cc +++ b/test/cpp/ext/gsm/metadata_exchange_test.cc @@ -30,6 +30,7 @@ #include "src/core/lib/channel/call_tracer.h" #include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gpr/tmpfile.h" #include "src/core/lib/gprpp/env.h" #include "src/cpp/ext/gsm/gsm_observability.h" #include "src/cpp/ext/otel/otel_plugin.h" @@ -43,29 +44,45 @@ namespace { class TestScenario { public: - enum class Type : std::uint8_t { kGke, kUnknown }; + enum class ResourceType : std::uint8_t { kGke, kUnknown }; + enum class XdsBootstrapSource : std::uint8_t { kFromFile, kFromConfig }; - explicit TestScenario(Type type) : type_(type) {} + explicit TestScenario(ResourceType type, XdsBootstrapSource bootstrap_source) + : type_(type), bootstrap_source_(bootstrap_source) {} opentelemetry::sdk::resource::Resource GetTestResource() const { switch (type_) { - case Type::kGke: + case ResourceType::kGke: return TestGkeResource(); - case Type::kUnknown: + case ResourceType::kUnknown: return TestUnknownResource(); } } static std::string Name(const ::testing::TestParamInfo& info) { + std::string ret_val; switch (info.param.type_) { - case Type::kGke: - return "gke"; - case Type::kUnknown: - return "unknown"; + case ResourceType::kGke: + ret_val += "Gke"; + break; + case ResourceType::kUnknown: + ret_val += "Unknown"; + break; } + switch (info.param.bootstrap_source_) { + case TestScenario::XdsBootstrapSource::kFromFile: + ret_val += "BootstrapFromFile"; + break; + case TestScenario::XdsBootstrapSource::kFromConfig: + ret_val += "BootstrapFromConfig"; + break; + } + return ret_val; } - Type type() const { return type_; } + ResourceType type() const { return type_; } + + XdsBootstrapSource bootstrap_source() const { return bootstrap_source_; } private: static opentelemetry::sdk::resource::Resource TestGkeResource() { @@ -86,7 +103,8 @@ class TestScenario { return opentelemetry::sdk::resource::Resource::Create(attributes); } - Type type_; + ResourceType type_; + XdsBootstrapSource bootstrap_source_; }; class MetadataExchangeTest @@ -94,6 +112,24 @@ class MetadataExchangeTest public ::testing::WithParamInterface { protected: void Init(const absl::flat_hash_set& metric_names) { + const char* kBootstrap = + "{\"node\": {\"id\": " + "\"projects/1234567890/networks/mesh:mesh-id/nodes/" + "01234567-89ab-4def-8123-456789abcdef\"}}"; + switch (GetParam().bootstrap_source()) { + case TestScenario::XdsBootstrapSource::kFromFile: { + ASSERT_EQ(bootstrap_file_name_, nullptr); + FILE* bootstrap_file = + gpr_tmpfile("gcp_observability_config", &bootstrap_file_name_); + fputs(kBootstrap, bootstrap_file); + fclose(bootstrap_file); + grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP", bootstrap_file_name_); + break; + } + case TestScenario::XdsBootstrapSource::kFromConfig: + grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP_CONFIG", kBootstrap); + break; + } OTelPluginEnd2EndTest::Init( metric_names, /*resource=*/GetParam().GetTestResource(), /*labels_injector=*/ @@ -101,47 +137,64 @@ class MetadataExchangeTest GetParam().GetTestResource().GetAttributes())); } - void VerifyGkeServiceMeshAttributes( + ~MetadataExchangeTest() override { + grpc_core::UnsetEnv("GRPC_GCP_OBSERVABILITY_CONFIG"); + grpc_core::UnsetEnv("GRPC_XDS_BOOTSTRAP"); + if (bootstrap_file_name_ != nullptr) { + remove(bootstrap_file_name_); + gpr_free(bootstrap_file_name_); + } + } + + void VerifyServiceMeshAttributes( const std::map& - attributes, - bool local_only = false) { - if (!local_only) { - switch (GetParam().type()) { - case TestScenario::Type::kGke: - EXPECT_EQ( - absl::get(attributes.at("gsm.remote_workload_type")), - "gcp_kubernetes_engine"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_pod_name")), - "pod"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_container_name")), - "container"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_namespace_name")), - "namespace"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_cluster_name")), - "cluster"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_location")), - "region"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_project_id")), - "id"); - EXPECT_EQ(absl::get( - attributes.at("gsm.remote_workload_canonical_service")), - "canonical_service"); - break; - case TestScenario::Type::kUnknown: - EXPECT_EQ( - absl::get(attributes.at("gsm.remote_workload_type")), - "random"); - break; - } + attributes) { + EXPECT_EQ(absl::get(attributes.at("gsm.mesh_id")), "mesh-id"); + switch (GetParam().type()) { + case TestScenario::ResourceType::kGke: + EXPECT_EQ( + absl::get(attributes.at("gsm.remote_workload_type")), + "gcp_kubernetes_engine"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_pod_name")), + "pod"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_container_name")), + "container"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_namespace_name")), + "namespace"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_cluster_name")), + "cluster"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_location")), + "region"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_project_id")), + "id"); + EXPECT_EQ(absl::get( + attributes.at("gsm.remote_workload_canonical_service")), + "canonical_service"); + break; + case TestScenario::ResourceType::kUnknown: + EXPECT_EQ( + absl::get(attributes.at("gsm.remote_workload_type")), + "random"); + break; } } + + void VerifyNoServiceMeshAttributes( + const std::map& + attributes) { + EXPECT_EQ(attributes.find("gsm.remote_workload_type"), attributes.end()); + } + + private: + char* bootstrap_file_name_ = nullptr; }; TEST_P(MetadataExchangeTest, ClientAttemptStarted) { @@ -165,7 +218,7 @@ TEST_P(MetadataExchangeTest, ClientAttemptStarted) { EXPECT_EQ(absl::get(attributes.at("grpc.method")), kMethodName); EXPECT_EQ(absl::get(attributes.at("grpc.target")), canonical_server_address_); - VerifyGkeServiceMeshAttributes(attributes, /*local_only=*/true); + VerifyNoServiceMeshAttributes(attributes); } TEST_P(MetadataExchangeTest, ClientAttemptDuration) { @@ -189,7 +242,27 @@ TEST_P(MetadataExchangeTest, ClientAttemptDuration) { EXPECT_EQ(absl::get(attributes.at("grpc.target")), canonical_server_address_); EXPECT_EQ(absl::get(attributes.at("grpc.status")), "OK"); - VerifyGkeServiceMeshAttributes(attributes); + VerifyServiceMeshAttributes(attributes); +} + +TEST_P(MetadataExchangeTest, ServerCallStarted) { + Init( + /*metric_names=*/{grpc::internal::OTelServerCallStartedInstrumentName()}); + SendRPC(); + const char* kMetricName = "grpc.server.call.started"; + auto data = ReadCurrentMetricsData( + [&](const absl::flat_hash_map< + std::string, + std::vector>& + data) { return !data.contains(kMetricName); }); + ASSERT_EQ(data[kMetricName].size(), 1); + auto point_data = absl::get_if( + &data[kMetricName][0].point_data); + ASSERT_NE(point_data, nullptr); + ASSERT_EQ(absl::get(point_data->value_), 1); + const auto& attributes = data[kMetricName][0].attributes.GetAttributes(); + EXPECT_EQ(absl::get(attributes.at("grpc.method")), kMethodName); + VerifyNoServiceMeshAttributes(attributes); } TEST_P(MetadataExchangeTest, ServerCallDuration) { @@ -211,13 +284,20 @@ TEST_P(MetadataExchangeTest, ServerCallDuration) { const auto& attributes = data[kMetricName][0].attributes.GetAttributes(); EXPECT_EQ(absl::get(attributes.at("grpc.method")), kMethodName); EXPECT_EQ(absl::get(attributes.at("grpc.status")), "OK"); - VerifyGkeServiceMeshAttributes(attributes); + VerifyServiceMeshAttributes(attributes); } INSTANTIATE_TEST_SUITE_P( MetadataExchange, MetadataExchangeTest, - ::testing::Values(TestScenario(TestScenario::Type::kGke), - TestScenario(TestScenario::Type::kUnknown)), + ::testing::Values( + TestScenario(TestScenario::ResourceType::kGke, + TestScenario::XdsBootstrapSource::kFromConfig), + TestScenario(TestScenario::ResourceType::kGke, + TestScenario::XdsBootstrapSource::kFromFile), + TestScenario(TestScenario::ResourceType::kUnknown, + TestScenario::XdsBootstrapSource::kFromConfig), + TestScenario(TestScenario::ResourceType::kUnknown, + TestScenario::XdsBootstrapSource::kFromFile)), &TestScenario::Name); } // namespace