From 003092a2020c927408fb173dfd9dcdf2dfc35775 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 21 Feb 2017 14:08:01 -0800 Subject: [PATCH 01/32] code and BUILD file for creating a grpcz client The client binary can be built with 'bazel build //tools/grpcz:grpcz_client' and can be invoked with bazel-bin/tools/grpcz/grpcz_client --server SERVER_ADDR:PORT You can see the stats page at http://localhost:8000/grpcz --- WORKSPACE | 26 +- tools/grpcz/BUILD | 73 +++++ tools/grpcz/any.proto | 139 +++++++++ tools/grpcz/census.proto | 318 +++++++++++++++++++++ tools/grpcz/empty.proto | 52 ++++ tools/grpcz/grpcz_client.cc | 174 +++++++++++ tools/grpcz/monitoring.proto | 131 +++++++++ tools/run_tests/sanity/check_submodules.sh | 2 +- 8 files changed, 910 insertions(+), 5 deletions(-) create mode 100644 tools/grpcz/BUILD create mode 100644 tools/grpcz/any.proto create mode 100644 tools/grpcz/census.proto create mode 100644 tools/grpcz/empty.proto create mode 100644 tools/grpcz/grpcz_client.cc create mode 100644 tools/grpcz/monitoring.proto diff --git a/WORKSPACE b/WORKSPACE index 4f90f06d881..5d163f78e81 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -33,26 +33,44 @@ bind( actual = "@submodule_gtest//:gtest", ) +bind( + name = "gflags", + actual = "@com_github_gflags_gflags//:gflags", +) + new_local_repository( name = "submodule_boringssl", - path = "third_party/boringssl-with-bazel", build_file = "third_party/boringssl-with-bazel/BUILD", + path = "third_party/boringssl-with-bazel", ) new_local_repository( name = "submodule_zlib", - path = "third_party/zlib", build_file = "third_party/zlib.BUILD", + path = "third_party/zlib", ) new_local_repository( name = "submodule_protobuf", - path = "third_party/protobuf", build_file = "third_party/protobuf/BUILD", + path = "third_party/protobuf", ) new_local_repository( name = "submodule_gtest", - path = "third_party/googletest", build_file = "third_party/gtest.BUILD", + path = "third_party/googletest", +) + +local_repository( + name = "com_github_gflags_gflags", + path = "third_party/gflags", +) +# used for tools/grpcz/grpcz_client +git_repository( + name = "mongoose_repo", + commit = "21b9ddd490783e3afaa0fa9b45d6c1133eb922dc", + remote = "https://github.com/makdharma/mongoose.git" ) + + diff --git a/tools/grpcz/BUILD b/tools/grpcz/BUILD new file mode 100644 index 00000000000..34bd072d004 --- /dev/null +++ b/tools/grpcz/BUILD @@ -0,0 +1,73 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +licenses(["notice"]) # 3-clause BSD + +package(default_visibility = ["//visibility:public"]) + +load("//:bazel/generate_cc.bzl", "generate_cc") + +proto_library ( + name = "monitoring_proto_local_copy", + srcs = [ + # TODO (erikgribkoff) : remove the local copies of these protos + "monitoring.proto", + "empty.proto", + "any.proto", + "census.proto", + ], +) + +generate_cc( + name = "monitoring_codegen", + srcs = [":monitoring_proto_local_copy"], +) + +generate_cc( + name = "monitoring_grpc_codegen", + srcs = [":monitoring_proto_local_copy"], + plugin = "//:grpc_cpp_plugin", +) + +cc_library( + name = "proto_lib", + srcs = [":monitoring_codegen", ":monitoring_grpc_codegen"], + hdrs = [":monitoring_codegen", ":monitoring_grpc_codegen"], + deps = ["//:grpc++", "//:grpc++_codegen_proto", "//external:protobuf"], +) + +cc_binary( + name = "grpcz_client", + srcs = ["grpcz_client.cc",], + deps = [ + "proto_lib", + "//external:gflags", + "@mongoose_repo//:mongoose_lib", + ], +) diff --git a/tools/grpcz/any.proto b/tools/grpcz/any.proto new file mode 100644 index 00000000000..a224a2b596e --- /dev/null +++ b/tools/grpcz/any.proto @@ -0,0 +1,139 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +//package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "github.com/golang/protobuf/ptypes/any"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// `Any` contains an arbitrary serialized protocol buffer message along with a +// URL that describes the type of the serialized message. +// +// Protobuf library provides support to pack/unpack Any values in the form +// of utility functions or additional generated methods of the Any type. +// +// Example 1: Pack and unpack a message in C++. +// +// Foo foo = ...; +// Any any; +// any.PackFrom(foo); +// ... +// if (any.UnpackTo(&foo)) { +// ... +// } +// +// Example 2: Pack and unpack a message in Java. +// +// Foo foo = ...; +// Any any = Any.pack(foo); +// ... +// if (any.is(Foo.class)) { +// foo = any.unpack(Foo.class); +// } +// +// Example 3: Pack and unpack a message in Python. +// +// foo = Foo(...) +// any = Any() +// any.Pack(foo) +// ... +// if any.Is(Foo.DESCRIPTOR): +// any.Unpack(foo) +// ... +// +// The pack methods provided by protobuf library will by default use +// 'type.googleapis.com/full.type.name' as the type URL and the unpack +// methods only use the fully qualified type name after the last '/' +// in the type URL, for example "foo.bar.com/x/y.z" will yield type +// name "y.z". +// +// +// JSON +// ==== +// The JSON representation of an `Any` value uses the regular +// representation of the deserialized, embedded message, with an +// additional field `@type` which contains the type URL. Example: +// +// package google.profile; +// message Person { +// string first_name = 1; +// string last_name = 2; +// } +// +// { +// "@type": "type.googleapis.com/google.profile.Person", +// "firstName": , +// "lastName": +// } +// +// If the embedded message type is well-known and has a custom JSON +// representation, that representation will be embedded adding a field +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): +// +// { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": "1.212s" +// } +// +message Any { + // A URL/resource name whose content describes the type of the + // serialized protocol buffer message. + // + // For URLs which use the scheme `http`, `https`, or no scheme, the + // following restrictions and interpretations apply: + // + // * If no scheme is provided, `https` is assumed. + // * The last segment of the URL's path must represent the fully + // qualified name of the type (as in `path/google.protobuf.Duration`). + // The name should be in a canonical form (e.g., leading "." is + // not accepted). + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] + // value in binary format, or produce an error. + // * Applications are allowed to cache lookup results based on the + // URL, or have them precompiled into a binary to avoid any + // lookup. Therefore, binary compatibility needs to be preserved + // on changes to types. (Use versioned type names to manage + // breaking changes.) + // + // Schemes other than `http`, `https` (or the empty scheme) might be + // used with implementation specific semantics. + // + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/tools/grpcz/census.proto b/tools/grpcz/census.proto new file mode 100644 index 00000000000..d1ff69400b0 --- /dev/null +++ b/tools/grpcz/census.proto @@ -0,0 +1,318 @@ +// Copyright 2017, Google Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//TODO(ericgribkoff) Depend on this directly from the instrumentation-proto +//repository. + +syntax = "proto3"; + +package google.instrumentation; + +option java_package = "com.google.instrumentation.stats.proto"; +option java_outer_classname = "CensusProto"; + +// All the census protos. +// +// Nomenclature notes: +// * Capitalized names below (like View) are protos. +// * Protos which describe types are named with a Descriptor suffix (e.g. +// MesurementDescriptor). +// +// Census lets you define the type and description of the data being measured +// (e.g. the latency of an RPC or the number of CPU cycles spent on an +// operation using MeasurementDescriptor. As individual measurements (a double +// value) for are recorded, they are aggregated together into an +// Aggregation. There are two Aggregation types available: Distribution +// (describes the distribution of all measurements, possibly with a histogram) +// and IntervalStats (the count and mean of measurements across specified time +// periods). An Aggregation is described by an AggregationDescriptor. +// +// You can define how your measurements (described by a MeasurementDescriptor) +// are broken down by Tag values and which Aggregations to use through a +// ViewDescriptor. The output (all measurements broken down by tag values into +// specific Aggregations) is called a View. + + +// The following two types are copied from +// google/protobuf/{duration,timestamp}.proto. Ideally, we would be able to +// import them, but this causes compilation issues on C-based systems +// (e.g. https://koti.kapsi.fi/jpa/nanopb/), which cannot process the C++ +// headers generated from the standard protobuf distribution. See the relevant +// proto files for full documentation of these types. + +message Duration { + // Signed seconds of the span of time. Must be from -315,576,000,000 + // to +315,576,000,000 inclusive. + int64 seconds = 1; + + // Signed fractions of a second at nanosecond resolution of the span + // of time. Durations less than one second are represented with a 0 + // `seconds` field and a positive or negative `nanos` field. For durations + // of one second or more, a non-zero value for the `nanos` field must be + // of the same sign as the `seconds` field. Must be from -999,999,999 + // to +999,999,999 inclusive. + int32 nanos = 2; +} + +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} + +// MeasurementDescriptor describes a data point (measurement) type. +message MeasurementDescriptor { + // A descriptive name, e.g. rpc_latency, cpu. Must be unique. + string name = 1; + + // More detailed description of the resource, used in documentation. + string description = 2; + + // Fundamental units of measurement supported by Census + // TODO(aveitch): expand this to include other S.I. units? + enum BasicUnit { + UNKNOWN = 0; // Implementations should not use this + SCALAR = 1; // Dimensionless + BITS = 2; // A single bit + BYTES = 3; // An 8-bit byte + SECONDS = 4; // S.I. unit + CORES = 5; // CPU core usage + MAX_UNITS = 6; // Last defined value; implementations should only use + // this for validation. + } + + // MeasurementUnit lets you build compound units of the form + // 10^n * (A * B * ...) / (X * Y * ...), + // where the elements in the numerator and denominator are all BasicUnits. A + // MeasurementUnit must have at least one BasicUnit in its numerator. + // + // To specify multiplication in the numerator or denominator, simply specify + // multiple numerator or denominator fields. For example: + // + // - byte-seconds (i.e. bytes * seconds): + // numerator: BYTES + // numerator: SECS + // + // - events/sec^2 (i.e. rate of change of events/sec): + // numerator: SCALAR + // denominator: SECS + // denominator: SECS + // + // To specify multiples (in power of 10) of units, specify a non-zero + // 'power10' value, for example: + // + // - MB/s (i.e. megabytes / s): + // power10: 6 + // numerator: BYTES + // denominator: SECS + // + // - nanoseconds + // power10: -9 + // numerator: SECS + message MeasurementUnit { + int32 power10 = 1; + repeated BasicUnit numerators = 2; + repeated BasicUnit denominators = 3; + } + + // The units used by this type of measurement. + MeasurementUnit unit = 3; +} + +// An aggregation summarizes a series of individual measurements. There are +// two types of aggregation (IntervalAggregation and DistributionAggregation), +// unique types of each can be set using descriptors for each. + +// DistributionAggregation contains summary statistics for a population of +// values and, optionally, a histogram representing the distribution of those +// values across a specified set of histogram buckets, as defined in +// DistributionAggregationDescriptor.bucket_bounds. +// +// The summary statistics are the count, mean, minimum, and the maximum of the +// set of population of values. +// +// Although it is not forbidden, it is generally a bad idea to include +// non-finite values (infinities or NaNs) in the population of values, as this +// will render the `mean` field meaningless. +message DistributionAggregation { + // The number of values in the population. Must be non-negative. + int64 count = 1; + + // The arithmetic mean of the values in the population. If `count` is zero + // then this field must be zero. + double mean = 2; + + // The sum of the values in the population. If `count` is zero then this + // field must be zero. + double sum = 3; + + // Describes a range of population values. + message Range { + // The minimum of the population values. + double min = 1; + // The maximum of the population values. + double max = 2; + } + + // The range of the population values. If `count` is zero, this field will not + // be defined. + Range range = 4; + + // A Distribution may optionally contain a histogram of the values in the + // population. The histogram is given in `bucket_count` as counts of values + // that fall into one of a sequence of non-overlapping buckets, as described + // by `DistributionAggregationDescriptor.bucket_boundaries`. The sum of the + // values in `bucket_counts` must equal the value in `count`. + // + // Bucket counts are given in order under the numbering scheme described + // above (the underflow bucket has number 0; the finite buckets, if any, + // have numbers 1 through N-2; the overflow bucket has number N-1). + // + // The size of `bucket_count` must be no greater than N as defined in + // `bucket_boundaries`. + // + // Any suffix of trailing zero bucket_count fields may be omitted. + repeated int64 bucket_counts = 5; + + // Tags associated with this DistributionAggregation. These will be filled + // in based on the View specification. + repeated Tag tags = 6; +} + +message DistributionAggregationDescriptor { + // A Distribution may optionally contain a histogram of the values in the + // population. The bucket boundaries for that histogram are described by + // `bucket_bounds`. This defines `size(bucket_bounds) + 1` (= N) + // buckets. The boundaries for bucket index i are: + // + // [-infinity, bucket_bounds[i]) for i == 0 + // [bucket_bounds[i-1], bucket_bounds[i]) for 0 < i < N-2 + // [bucket_bounds[i-1], +infinity) for i == N-1 + // + // i.e. an underflow bucket (number 0), zero or more finite buckets (1 + // through N - 2, and an overflow bucket (N - 1), with inclusive lower + // bounds and exclusive upper bounds. + // + // If `bucket_bounds` has no elements (zero size), then there is no + // histogram associated with the Distribution. If `bucket_bounds` has only + // one element, there are no finite buckets, and that single element is the + // common boundary of the overflow and underflow buckets. The values must + // be monotonically increasing. + repeated double bucket_bounds = 1; +} + +// An IntervalAggreation records summary stats over various time +// windows. These stats are approximate, with the degree of accuracy +// controlled by setting the n_sub_intervals parameter in the +// IntervalAggregationDescriptor. +message IntervalAggregation { + // Summary statistic over a single time interval. + message Interval { + // The interval duration. Must be positive. + Duration interval_size = 1; + // Approximate number of measurements recorded in this interval. + double count = 2; + // The cumulative sum of measurements in this interval. + double sum = 3; + } + + // Full set of intervals for this aggregation. + repeated Interval intervals = 1; + + // Tags associated with this IntervalAggregation. These will be filled in + // based on the View specification. + repeated Tag tags = 2; +} + +// An IntervalAggreationDescriptor specifies time intervals for an +// IntervalAggregation. +message IntervalAggregationDescriptor { + // Number of internal sub-intervals to use when collecting stats for each + // interval. The max error in interval measurements will be approximately + // 1/n_sub_intervals (although in practice, this will only be approached in + // the presence of very large and bursty workload changes), and underlying + // memory usage will be roughly proportional to the value of this + // field. Must be in the range [2, 20]. A value of 5 will be used if this is + // unspecified. + int32 n_sub_intervals = 1; + + // The size of each interval, as a time duration. Must have at least one + // element. + repeated Duration interval_sizes = 2; +} + +// A Tag: key-value pair. +message Tag { + string key = 1; + string value = 2; +} + +// A ViewDescriptor specifies an AggregationDescriptor and a set of tag +// keys. Views instantiated from this descriptor will contain Aggregations +// broken down by the unique set of matching tag values for each measurement. +message ViewDescriptor { + // Name of view. Must be unique. + string name = 1; + + // More detailed description, for documentation purposes. + string description = 2; + + // Name of a MeasurementDescriptor to be used for this view. + string measurement_descriptor_name = 3; + + // Aggregation type to associate with View. + oneof aggregation { + IntervalAggregationDescriptor interval_aggregation = 4; + DistributionAggregationDescriptor distribution_aggregation = 5; + } + + // Tag keys to match with a given measurement. If no keys are specified, + // then all stats are recorded. Keys must be unique. + repeated string tag_keys = 6; +} + +// DistributionView contains all aggregations for a view specified using a +// DistributionAggregationDescriptor. +message DistributionView { + // Aggregations - each will have a unique set of tag values for the tag_keys + // associated with the corresponding View. + repeated DistributionAggregation aggregations = 1; + + // Start and end timestamps over which aggregations was accumulated. + Timestamp start = 2; + Timestamp end = 3; +} + +// IntervalView contains all aggregations for a view specified using a +// IntervalAggregationDescriptor. +message IntervalView { + // Aggregations - each will have a unique set of tag values for the tag_keys + // associated with the corresponding View. + repeated IntervalAggregation aggregations = 1; +} + +// A View contains the aggregations based on a ViewDescriptor. +message View { + // ViewDescriptor name associated with this set of View. + string view_name = 1; + + oneof view { + DistributionView distribution_view = 2; + IntervalView interval_view = 3; + } +} diff --git a/tools/grpcz/empty.proto b/tools/grpcz/empty.proto new file mode 100644 index 00000000000..03cacd23308 --- /dev/null +++ b/tools/grpcz/empty.proto @@ -0,0 +1,52 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "github.com/golang/protobuf/ptypes/empty"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +// A generic empty message that you can re-use to avoid defining duplicated +// empty messages in your APIs. A typical example is to use it as the request +// or the response type of an API method. For instance: +// +// service Foo { +// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); +// } +// +// The JSON representation for `Empty` is empty JSON object `{}`. +message Empty {} diff --git a/tools/grpcz/grpcz_client.cc b/tools/grpcz/grpcz_client.cc new file mode 100644 index 00000000000..afca3b0532f --- /dev/null +++ b/tools/grpcz/grpcz_client.cc @@ -0,0 +1,174 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include +#include +#include + +#include "gflags/gflags.h" +#include "mongoose.h" + +// TODO (makdharma): remove local copies of these protos +#include "tools/grpcz/census.grpc.pb.h" +#include "tools/grpcz/monitoring.grpc.pb.h" + +DEFINE_string(server, "127.0.0.1:50052", + "file path (or host:port) where grpcz server is running"); +DEFINE_string(http_port, "8000", + "Port id for accessing the HTTP server that renders /grpcz page"); +DEFINE_bool(print, false, "only print the output and quit"); + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; + +using ::grpc::instrumentation::v1alpha::CanonicalRpcStats; +using ::grpc::instrumentation::v1alpha::Monitoring; + +static const std::string static_html_header = + " \ +\ +

GRPCZ FTW

\ + "; + +class GrpczClient { + public: + GrpczClient(std::shared_ptr channel) + : stub_(Monitoring::NewStub(channel)) {} + + std::string GetStatsAsJson() { + const ::google::protobuf::Empty request; + CanonicalRpcStats reply; + ClientContext context; + Status status = stub_->GetCanonicalRpcStats(&context, request, &reply); + + if (status.ok()) { + std::string json_str; + ::google::protobuf::util::MessageToJsonString(reply, &json_str); + return json_str; + } else { + static const std::string error_message_json = + "{\"grpcz Access Error\"\ + :{\"view\":{\"viewName\":\"grpcz Access Error\",\ + \"intervalView\":\"Server not running?\"}}}"; + std::cout << status.error_code() << ":= " << status.error_message() + << std::endl; + return error_message_json; + } + } + + private: + std::unique_ptr stub_; +}; + +static struct mg_serve_http_opts s_http_server_opts; +std::unique_ptr g_grpcz_client; + +static void ev_handler(struct mg_connection *nc, int ev, void *p) { + if (ev == MG_EV_HTTP_REQUEST) { + mg_serve_http(nc, (struct http_message *)p, s_http_server_opts); + } +} + +static void grpcz_handler(struct mg_connection *nc, int ev, void *ev_data) { + (void)ev; + (void)ev_data; + gpr_log(GPR_INFO, "fetching grpcz stats from %s", FLAGS_server.c_str()); + std::string json_str = g_grpcz_client->GetStatsAsJson(); + std::string rendered_html = + static_html_header + json_str + static_html_footer; + mg_printf(nc, "HTTP/1.0 200 OK\r\n\r\n%s", rendered_html.c_str()); + nc->flags |= MG_F_SEND_AND_CLOSE; +} + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + + // Create a client + g_grpcz_client.reset(new GrpczClient( + grpc::CreateChannel(FLAGS_server, grpc::InsecureChannelCredentials()))); + if (FLAGS_print) { + g_grpcz_client->GetStatsAsJson(); + return 0; + } + + // Set up a mongoose webserver handler + struct mg_mgr mgr; + mg_mgr_init(&mgr, NULL); + gpr_log(GPR_INFO, "Starting grpcz web server on port %s\n", + FLAGS_http_port.c_str()); + + struct mg_connection *nc = mg_bind(&mgr, FLAGS_http_port.c_str(), ev_handler); + if (nc == NULL) { + gpr_log(GPR_ERROR, "Failed to create listener on port %s\n", + FLAGS_http_port.c_str()); + return -11; + } + mg_register_http_endpoint(nc, "/grpcz", grpcz_handler); + mg_set_protocol_http_websocket(nc); + + // Poll in a loop and serve /grpcz pages + for (;;) { + mg_mgr_poll(&mgr, 100); + } + mg_mgr_free(&mgr); + return 0; +} diff --git a/tools/grpcz/monitoring.proto b/tools/grpcz/monitoring.proto new file mode 100644 index 00000000000..ec3dfe9f74c --- /dev/null +++ b/tools/grpcz/monitoring.proto @@ -0,0 +1,131 @@ +// Copyright 2017, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. +syntax = "proto3"; + +// TODO(ericgribkoff) Figure out how to manage the external Census proto +// dependency. +import "tools/grpcz/census.proto"; +import "tools/grpcz/empty.proto"; +import "tools/grpcz/any.proto"; + +package grpc.instrumentation.v1alpha; + +option java_multiple_files = true; +option java_package = "io.grpc.instrumentation.v1alpha"; +option java_outer_classname = "MonitoringProto"; + +service Monitoring { + // Return canonical RPC stats + rpc GetCanonicalRpcStats(google.protobuf.Empty) returns (CanonicalRpcStats) { + } + + // Query the server for specific stats + rpc GetStats(StatsRequest) returns (StatsResponse) { + // TODO(aveitch, ericgribkoff): Pease define the stats response message + // StatsRequest would specifically identify the stats to be returned. + } + + // Request the server to stream back snapshots of the requested stats + rpc WatchStats(StatsRequest) returns (stream StatsResponse) { + } + + + // Return request traces. + rpc GetRequestTraces(TraceRequest) returns(TraceResponse) { + // TODO(aveitch): Please define the messages here + } + + // Return application-defined groups of monitoring data. + // This is a low level facility to allow extension of the monitoring API to + // application-specific monitoring data. Frameworks may use this to define + // additional groups of monitoring data made available by servers. + rpc GetCustomMonitoringData(MonitoringDataGroup) + returns (CustomMonitoringData) { + } + +} + +// Canonical RPC stats exported by gRPC. +message CanonicalRpcStats { + // Wrapper combining View and ViewDescriptor. + message View { + google.instrumentation.MeasurementDescriptor measurement_descriptor = 1; + google.instrumentation.ViewDescriptor view_descriptor = 2; + google.instrumentation.View view = 3; + } + + View rpc_client_errors = 1; + View rpc_client_completed_rpcs = 2; + View rpc_client_started_rpcs = 3; + View rpc_client_elapsed_time = 4; + View rpc_client_server_elapsed_time = 5; + View rpc_client_request_bytes = 6; + View rpc_client_response_bytes = 7; + View rpc_client_request_count = 8; + View rpc_client_response_count = 9; + View rpc_server_errors = 10; + View rpc_server_completed_rpcs = 11; + View rpc_server_server_elapsed_time = 12; + View rpc_server_request_bytes = 13; + View rpc_server_response_bytes = 14; + View rpc_server_request_count = 15; + View rpc_server_response_count = 16; + View rpc_server_elapsed_time = 17; + //TODO(ericgribkoff) Add minute-hour interval stats. +} + +message StatsRequest { + // TODO(aveitch): Complete definition of this type +} + +message StatsResponse { + // TODO(aveitch): Complete definition of this type +} + +message TraceRequest { + // TODO(aveitch): Complete definition of this type +} + +message TraceResponse { + // TODO(aveitch): Complete definition of this type +} + +message MonitoringDataGroup { + string name = 1; // name of a group of monitoring data +} + +// The wrapper for custom monitoring data. +message CustomMonitoringData { + // can be any application specific monitoring data + Any contents = 1; +} + diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 0b68319d290..cfe4e2731c0 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules 44c25c892a6229b20db7cd9dc05584ea865896de third_party/benchmark (v0.1.0-343-g44c25c8) 78684e5b222645828ca302e56b40b9daff2b2d27 third_party/boringssl (78684e5) 886e7d75368e3f4fab3f4d0d3584e4abfc557755 third_party/boringssl-with-bazel (version_for_cocoapods_7.0-857-g886e7d7) - f8a0efe03aa69b3336d8e228b37d4ccb17324b88 third_party/gflags (v2.2.0) + 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) a428e42072765993ff674fda72863c9f1aa2d268 third_party/protobuf (v3.1.0-alpha-1) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift (bcad917) From d1f3082ff608a82040d122ad3634942ac1515cf2 Mon Sep 17 00:00:00 2001 From: Nicolas 'Pixel' Noble Date: Tue, 21 Feb 2017 15:04:26 -0800 Subject: [PATCH 02/32] Adding shim for debugging node tests using vscode. --- .vscode/launch.json | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000000..700c61cceae --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,30 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Mocha Tests", + "cwd": "${workspaceRoot}", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha", + "windows": { + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha.cmd" + }, + "runtimeArgs": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceRoot}/src/node/test" + ], + "internalConsoleOptions": "openOnSessionStart" + }, + { + "type": "node", + "request": "attach", + "name": "Attach to Process", + "port": 5858 + } + ] +} From 86db6873db5b4f09a2a3d6b430db27ac0fd40445 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 22 Feb 2017 10:28:26 -0800 Subject: [PATCH 03/32] s/erik/eric --- tools/grpcz/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/grpcz/BUILD b/tools/grpcz/BUILD index 34bd072d004..4ae12d50064 100644 --- a/tools/grpcz/BUILD +++ b/tools/grpcz/BUILD @@ -36,7 +36,7 @@ load("//:bazel/generate_cc.bzl", "generate_cc") proto_library ( name = "monitoring_proto_local_copy", srcs = [ - # TODO (erikgribkoff) : remove the local copies of these protos + # TODO (ericgribkoff) : remove the local copies of these protos "monitoring.proto", "empty.proto", "any.proto", From 4be62589004152e7575ee1808fd4f52d9134296c Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 22 Feb 2017 14:32:41 -0800 Subject: [PATCH 04/32] updated to latest mongoose BUILD --- WORKSPACE | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 13786ce468c..f27159d4afd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -70,8 +70,6 @@ local_repository( # used for tools/grpcz/grpcz_client git_repository( name = "mongoose_repo", - commit = "21b9ddd490783e3afaa0fa9b45d6c1133eb922dc", + commit = "4120a97945b41195a6223a600dae8e3b19bed19e", remote = "https://github.com/makdharma/mongoose.git" ) - - From 38c10bd6d94a77a34b045ddb6511969c175248f7 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 1 Mar 2017 23:37:07 -0800 Subject: [PATCH 05/32] Avoid variable length arrays --- test/core/iomgr/ev_epoll_linux_test.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 4ec959995b2..867835aa1c6 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -144,10 +144,10 @@ static void test_pollset_queue_merge_items() { const int num_fds = 2; const int num_pollsets = 2; const int num_closures = 4; - test_fd tfds[num_fds]; - int fds[num_fds]; - test_pollset pollsets[num_pollsets]; - grpc_closure closures[num_closures]; + test_fd tfds[2 /* num_fds */]; + int fds[2 /* num_fds */]; + test_pollset pollsets[2 /* num_pollsets */]; + grpc_closure closures[4 /* num_closures */]; int i; int result = 0; @@ -217,9 +217,9 @@ static void test_add_fd_to_pollset() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; const int num_fds = 8; const int num_pollsets = 4; - test_fd tfds[num_fds]; - int fds[num_fds]; - test_pollset pollsets[num_pollsets]; + test_fd tfds[8 /* num_fds */]; + int fds[8 /* num_fds */]; + test_pollset pollsets[4 /* num_pollsets */]; void *expected_pi = NULL; int i; From 5ede0e07a792627fb129522254937bfc4b3a73db Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 2 Mar 2017 15:49:09 -0800 Subject: [PATCH 06/32] Use macros instead of const variables --- test/core/iomgr/ev_epoll_linux_test.c | 63 ++++++++++++++++----------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 867835aa1c6..ba6362e3954 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -139,23 +139,25 @@ static void increment(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { * polling_island_merge()[ev_epoll_linux.c], where the parent relationship was * inverted. */ + +#define NUM_FDS 2 +#define NUM_POLLSETS 2 +#define NUM_CLOSURES 4 + static void test_pollset_queue_merge_items() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - const int num_fds = 2; - const int num_pollsets = 2; - const int num_closures = 4; - test_fd tfds[2 /* num_fds */]; - int fds[2 /* num_fds */]; - test_pollset pollsets[2 /* num_pollsets */]; - grpc_closure closures[4 /* num_closures */]; + test_fd tfds[NUM_FDS]; + int fds[NUM_FDS]; + test_pollset pollsets[NUM_POLLSETS]; + grpc_closure closures[NUM_CLOSURES]; int i; int result = 0; - test_fd_init(tfds, fds, num_fds); - test_pollset_init(pollsets, num_pollsets); + test_fd_init(tfds, fds, NUM_FDS); + test_pollset_init(pollsets, NUM_POLLSETS); /* Two distinct polling islands, each with their own FD and pollset. */ - for (i = 0; i < num_fds; i++) { + for (i = 0; i < NUM_FDS; i++) { grpc_pollset_add_fd(&exec_ctx, pollsets[i].pollset, tfds[i].fd); grpc_exec_ctx_flush(&exec_ctx); } @@ -173,7 +175,7 @@ static void test_pollset_queue_merge_items() { grpc_closure_init( closures + 3, increment, &result, grpc_workqueue_scheduler(grpc_fd_get_polling_island(tfds[1].fd))); - for (i = 0; i < num_closures; ++i) { + for (i = 0; i < NUM_CLOSURES; ++i) { grpc_closure_sched(&exec_ctx, closures + i, GRPC_ERROR_NONE); } @@ -186,7 +188,7 @@ static void test_pollset_queue_merge_items() { * the merged polling island. */ grpc_pollset_worker *worker = NULL; - for (i = 0; i < num_closures; ++i) { + for (i = 0; i < NUM_CLOSURES; ++i) { const gpr_timespec deadline = gpr_time_add( gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(2, GPR_TIMESPAN)); gpr_mu_lock(pollsets[1].mu); @@ -196,13 +198,17 @@ static void test_pollset_queue_merge_items() { gpr_now(GPR_CLOCK_MONOTONIC), deadline)); gpr_mu_unlock(pollsets[1].mu); } - GPR_ASSERT(result == num_closures); + GPR_ASSERT(result == NUM_CLOSURES); - test_fd_cleanup(&exec_ctx, tfds, num_fds); - test_pollset_cleanup(&exec_ctx, pollsets, num_pollsets); + test_fd_cleanup(&exec_ctx, tfds, NUM_FDS); + test_pollset_cleanup(&exec_ctx, pollsets, NUM_POLLSETS); grpc_exec_ctx_finish(&exec_ctx); } +#undef NUM_FDS +#undef NUM_POLLSETS +#undef NUM_CLOSURES + /* * Cases to test: * case 1) Polling islands of both fd and pollset are NULL @@ -213,18 +219,20 @@ static void test_pollset_queue_merge_items() { * case 4.2) Polling islands of fd and pollset are NOT-equal (This results * in a merge) * */ + +#define NUM_FDS 8 +#define NUM_POLLSETS 4 + static void test_add_fd_to_pollset() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - const int num_fds = 8; - const int num_pollsets = 4; - test_fd tfds[8 /* num_fds */]; - int fds[8 /* num_fds */]; - test_pollset pollsets[4 /* num_pollsets */]; + test_fd tfds[NUM_FDS]; + int fds[NUM_FDS]; + test_pollset pollsets[NUM_POLLSETS]; void *expected_pi = NULL; int i; - test_fd_init(tfds, fds, num_fds); - test_pollset_init(pollsets, num_pollsets); + test_fd_init(tfds, fds, NUM_FDS); + test_pollset_init(pollsets, NUM_POLLSETS); /*Step 1. * Create three polling islands (This will exercise test case 1 and 2) with @@ -285,22 +293,25 @@ static void test_add_fd_to_pollset() { /* Compare Fd:0's polling island with that of all other Fds */ expected_pi = grpc_fd_get_polling_island(tfds[0].fd); - for (i = 1; i < num_fds; i++) { + for (i = 1; i < NUM_FDS; i++) { GPR_ASSERT(grpc_are_polling_islands_equal( expected_pi, grpc_fd_get_polling_island(tfds[i].fd))); } /* Compare Fd:0's polling island with that of all other pollsets */ - for (i = 0; i < num_pollsets; i++) { + for (i = 0; i < NUM_POLLSETS; i++) { GPR_ASSERT(grpc_are_polling_islands_equal( expected_pi, grpc_pollset_get_polling_island(pollsets[i].pollset))); } - test_fd_cleanup(&exec_ctx, tfds, num_fds); - test_pollset_cleanup(&exec_ctx, pollsets, num_pollsets); + test_fd_cleanup(&exec_ctx, tfds, NUM_FDS); + test_pollset_cleanup(&exec_ctx, pollsets, NUM_POLLSETS); grpc_exec_ctx_finish(&exec_ctx); } +#undef NUM_FDS +#undef NUM_POLLSETS + int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); From 4166dc3536986c5295131a6e7c5920baaae487b3 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 9 Mar 2017 14:30:18 -0800 Subject: [PATCH 07/32] Add script to allow Jenkins to comment on PRs --- tools/jenkins/comment_on_pr.sh | 45 ++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tools/jenkins/comment_on_pr.sh diff --git a/tools/jenkins/comment_on_pr.sh b/tools/jenkins/comment_on_pr.sh new file mode 100644 index 00000000000..79c54c2159c --- /dev/null +++ b/tools/jenkins/comment_on_pr.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins to comment $1 on pull requests +# when triggered by a build + +set -e + +if [ -z $1 ] || [ -z $JENKINS_OAUTH_TOKEN ] || [ -z $ghprbPullId ]; then + echo "Insufficient arguments or environment variables provided." + exit 1 +fi + +# Format the comment message to JSON +COMMENT_MESSAGE="{\"body\":\"$1\"}" + +curl -k -H "Authorization: token $JENKINS_OAUTH_TOKEN" -H "Content-Type: application/json" \ + -d "$COMMENT_MESSAGE" https://api.github.com/repos/grpc/grpc/issues/$ghprbPullId/comments From aff1c05ed78e52f41c8b1e7c03b3c57f08368130 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Thu, 9 Mar 2017 15:08:01 -0800 Subject: [PATCH 08/32] Make Jenkins post microbenchmarking diff in a comment --- tools/jenkins/comment_on_pr.sh | 2 +- tools/run_tests/run_microbenchmark.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) mode change 100644 => 100755 tools/jenkins/comment_on_pr.sh diff --git a/tools/jenkins/comment_on_pr.sh b/tools/jenkins/comment_on_pr.sh old mode 100644 new mode 100755 index 79c54c2159c..85f33aa917a --- a/tools/jenkins/comment_on_pr.sh +++ b/tools/jenkins/comment_on_pr.sh @@ -33,7 +33,7 @@ set -e -if [ -z $1 ] || [ -z $JENKINS_OAUTH_TOKEN ] || [ -z $ghprbPullId ]; then +if [ -z "$1" ] || [ -z $JENKINS_OAUTH_TOKEN ] || [ -z $ghprbPullId ]; then echo "Insufficient arguments or environment variables provided." exit 1 fi diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 57b2636e569..12d98158a0e 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -229,6 +229,8 @@ argp.add_argument('--summary_time', type=int, help='Minimum time to run benchmarks for the summary collection') args = argp.parse_args() +if args.diff_perf: + git_comment = '' try: for collect in args.collect: @@ -262,7 +264,14 @@ try: if diff: heading('Performance diff: %s' % bm_name) text(diff) + git_comment += '```\\nPerformance diff: %s\\n%s\\n```\\n' % (bm_name, diff.replace('\n', '\\n')) finally: + if args.diff_perf: + subprocess.call(['tools/jenkins/comment_on_pr.sh "%s"' % git_comment.replace('`', '\`')], + stdout=subprocess.PIPE, + shell=True) + if not os.path.exists('reports'): + os.makedirs('reports') index_html += "\n\n" with open('reports/index.html', 'w') as f: f.write(index_html) From c2a8a8f5c248516c684136b22181f2db204df7bc Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 17 Mar 2017 11:10:08 -0700 Subject: [PATCH 09/32] addressed review feedback using new grpc_build_system, and cleaned up typos. --- BUILD | 2 + tools/grpcz/BUILD | 38 ++++------ tools/grpcz/any.proto | 139 ----------------------------------- tools/grpcz/empty.proto | 52 ------------- tools/grpcz/grpcz_client.cc | 41 +++++++---- tools/grpcz/monitoring.proto | 6 +- 6 files changed, 44 insertions(+), 234 deletions(-) delete mode 100644 tools/grpcz/any.proto delete mode 100644 tools/grpcz/empty.proto diff --git a/BUILD b/BUILD index ca0a1c56076..5a2e7a72e9e 100644 --- a/BUILD +++ b/BUILD @@ -1132,6 +1132,7 @@ grpc_cc_library( "src/cpp/common/rpc_method.cc", "src/cpp/common/version_cc.cc", "src/cpp/server/async_generic_service.cc", + "src/cpp/server/channel_argument_option.cc", "src/cpp/server/create_default_thread_pool.cc", "src/cpp/server/dynamic_thread_pool.cc", "src/cpp/server/health/default_health_check_service.cc", @@ -1173,6 +1174,7 @@ grpc_cc_library( "include/grpc++/grpc++.h", "include/grpc++/health_check_service_interface.h", "include/grpc++/impl/call.h", + "include/grpc++/impl/channel_argument_option.h", "include/grpc++/impl/client_unary_call.h", "include/grpc++/impl/codegen/core_codegen.h", "include/grpc++/impl/grpc_library.h", diff --git a/tools/grpcz/BUILD b/tools/grpcz/BUILD index 4ae12d50064..5e1faf7064f 100644 --- a/tools/grpcz/BUILD +++ b/tools/grpcz/BUILD @@ -31,43 +31,33 @@ licenses(["notice"]) # 3-clause BSD package(default_visibility = ["//visibility:public"]) -load("//:bazel/generate_cc.bzl", "generate_cc") +load("//:bazel/grpc_build_system.bzl", "grpc_proto_library") -proto_library ( - name = "monitoring_proto_local_copy", +grpc_proto_library ( + name = "monitoring_proto", srcs = [ - # TODO (ericgribkoff) : remove the local copies of these protos "monitoring.proto", - "empty.proto", - "any.proto", - "census.proto", ], + deps = [ + ":census_proto", + ], + well_known_protos = "@submodule_protobuf//:well_known_protos", ) -generate_cc( - name = "monitoring_codegen", - srcs = [":monitoring_proto_local_copy"], -) - -generate_cc( - name = "monitoring_grpc_codegen", - srcs = [":monitoring_proto_local_copy"], - plugin = "//:grpc_cpp_plugin", -) - -cc_library( - name = "proto_lib", - srcs = [":monitoring_codegen", ":monitoring_grpc_codegen"], - hdrs = [":monitoring_codegen", ":monitoring_grpc_codegen"], - deps = ["//:grpc++", "//:grpc++_codegen_proto", "//external:protobuf"], +grpc_proto_library ( + name = "census_proto", + srcs = [ + "census.proto", + ], + well_known_protos = "@submodule_protobuf//:well_known_protos", ) cc_binary( name = "grpcz_client", srcs = ["grpcz_client.cc",], deps = [ - "proto_lib", "//external:gflags", + "monitoring_proto", "@mongoose_repo//:mongoose_lib", ], ) diff --git a/tools/grpcz/any.proto b/tools/grpcz/any.proto deleted file mode 100644 index a224a2b596e..00000000000 --- a/tools/grpcz/any.proto +++ /dev/null @@ -1,139 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -//package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option go_package = "github.com/golang/protobuf/ptypes/any"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "AnyProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; - -// `Any` contains an arbitrary serialized protocol buffer message along with a -// URL that describes the type of the serialized message. -// -// Protobuf library provides support to pack/unpack Any values in the form -// of utility functions or additional generated methods of the Any type. -// -// Example 1: Pack and unpack a message in C++. -// -// Foo foo = ...; -// Any any; -// any.PackFrom(foo); -// ... -// if (any.UnpackTo(&foo)) { -// ... -// } -// -// Example 2: Pack and unpack a message in Java. -// -// Foo foo = ...; -// Any any = Any.pack(foo); -// ... -// if (any.is(Foo.class)) { -// foo = any.unpack(Foo.class); -// } -// -// Example 3: Pack and unpack a message in Python. -// -// foo = Foo(...) -// any = Any() -// any.Pack(foo) -// ... -// if any.Is(Foo.DESCRIPTOR): -// any.Unpack(foo) -// ... -// -// The pack methods provided by protobuf library will by default use -// 'type.googleapis.com/full.type.name' as the type URL and the unpack -// methods only use the fully qualified type name after the last '/' -// in the type URL, for example "foo.bar.com/x/y.z" will yield type -// name "y.z". -// -// -// JSON -// ==== -// The JSON representation of an `Any` value uses the regular -// representation of the deserialized, embedded message, with an -// additional field `@type` which contains the type URL. Example: -// -// package google.profile; -// message Person { -// string first_name = 1; -// string last_name = 2; -// } -// -// { -// "@type": "type.googleapis.com/google.profile.Person", -// "firstName": , -// "lastName": -// } -// -// If the embedded message type is well-known and has a custom JSON -// representation, that representation will be embedded adding a field -// `value` which holds the custom JSON in addition to the `@type` -// field. Example (for message [google.protobuf.Duration][]): -// -// { -// "@type": "type.googleapis.com/google.protobuf.Duration", -// "value": "1.212s" -// } -// -message Any { - // A URL/resource name whose content describes the type of the - // serialized protocol buffer message. - // - // For URLs which use the scheme `http`, `https`, or no scheme, the - // following restrictions and interpretations apply: - // - // * If no scheme is provided, `https` is assumed. - // * The last segment of the URL's path must represent the fully - // qualified name of the type (as in `path/google.protobuf.Duration`). - // The name should be in a canonical form (e.g., leading "." is - // not accepted). - // * An HTTP GET on the URL must yield a [google.protobuf.Type][] - // value in binary format, or produce an error. - // * Applications are allowed to cache lookup results based on the - // URL, or have them precompiled into a binary to avoid any - // lookup. Therefore, binary compatibility needs to be preserved - // on changes to types. (Use versioned type names to manage - // breaking changes.) - // - // Schemes other than `http`, `https` (or the empty scheme) might be - // used with implementation specific semantics. - // - string type_url = 1; - - // Must be a valid serialized protocol buffer of the above specified type. - bytes value = 2; -} diff --git a/tools/grpcz/empty.proto b/tools/grpcz/empty.proto deleted file mode 100644 index 03cacd23308..00000000000 --- a/tools/grpcz/empty.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -package google.protobuf; - -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; -option go_package = "github.com/golang/protobuf/ptypes/empty"; -option java_package = "com.google.protobuf"; -option java_outer_classname = "EmptyProto"; -option java_multiple_files = true; -option objc_class_prefix = "GPB"; -option cc_enable_arenas = true; - -// A generic empty message that you can re-use to avoid defining duplicated -// empty messages in your APIs. A typical example is to use it as the request -// or the response type of an API method. For instance: -// -// service Foo { -// rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); -// } -// -// The JSON representation for `Empty` is empty JSON object `{}`. -message Empty {} diff --git a/tools/grpcz/grpcz_client.cc b/tools/grpcz/grpcz_client.cc index afca3b0532f..f4611466c53 100644 --- a/tools/grpcz/grpcz_client.cc +++ b/tools/grpcz/grpcz_client.cc @@ -46,11 +46,14 @@ #include "tools/grpcz/census.grpc.pb.h" #include "tools/grpcz/monitoring.grpc.pb.h" -DEFINE_string(server, "127.0.0.1:50052", - "file path (or host:port) where grpcz server is running"); +DEFINE_string( + grpcz_server, "127.0.0.1:8080", + "Unix domain socket path (e.g. unix://tmp/grpcz.sock) or IP address" + "(host:port) where grpcz server is running."); DEFINE_string(http_port, "8000", "Port id for accessing the HTTP server that renders /grpcz page"); -DEFINE_bool(print, false, "only print the output and quit"); +DEFINE_bool(print_to_console, false, + "print the JSON retreived from grpcz server and quit"); using grpc::Channel; using grpc::ClientContext; @@ -68,11 +71,14 @@ table, td, th { border: 1px solid black; } \ static const std::string static_html_footer = "' class='hidden'>\ -

GRPCZ FTW

\ +

GRPCZ Statistics

\ "; class GrpczClient { @@ -106,11 +113,9 @@ class GrpczClient { return json_str; } else { static const std::string error_message_json = - "{\"grpcz Access Error\"\ - :{\"view\":{\"viewName\":\"grpcz Access Error\",\ - \"intervalView\":\"Server not running?\"}}}"; - std::cout << status.error_code() << ":= " << status.error_message() - << std::endl; + "{\"Error Message\":\"" + status.error_message() + "\"}"; + gpr_log(GPR_DEBUG, "%d: %s", status.error_code(), + status.error_message().c_str()); return error_message_json; } } @@ -131,7 +136,7 @@ static void ev_handler(struct mg_connection *nc, int ev, void *p) { static void grpcz_handler(struct mg_connection *nc, int ev, void *ev_data) { (void)ev; (void)ev_data; - gpr_log(GPR_INFO, "fetching grpcz stats from %s", FLAGS_server.c_str()); + gpr_log(GPR_INFO, "fetching grpcz stats from %s", FLAGS_grpcz_server.c_str()); std::string json_str = g_grpcz_client->GetStatsAsJson(); std::string rendered_html = static_html_header + json_str + static_html_footer; @@ -143,10 +148,13 @@ int main(int argc, char **argv) { gflags::ParseCommandLineFlags(&argc, &argv, true); // Create a client - g_grpcz_client.reset(new GrpczClient( - grpc::CreateChannel(FLAGS_server, grpc::InsecureChannelCredentials()))); - if (FLAGS_print) { - g_grpcz_client->GetStatsAsJson(); + g_grpcz_client.reset(new GrpczClient(grpc::CreateChannel( + FLAGS_grpcz_server, grpc::InsecureChannelCredentials()))); + if (FLAGS_print_to_console) { + // using GPR_ERROR since this is the default verbosity. _DEBUG or _INFO + // won't print unless GRPC_VERBOSITY env var is set appropriately, which + // might confuse users of this utility. + gpr_log(GPR_ERROR, "%s\n", g_grpcz_client->GetStatsAsJson().c_str()); return 0; } @@ -160,14 +168,15 @@ int main(int argc, char **argv) { if (nc == NULL) { gpr_log(GPR_ERROR, "Failed to create listener on port %s\n", FLAGS_http_port.c_str()); - return -11; + return -1; } mg_register_http_endpoint(nc, "/grpcz", grpcz_handler); mg_set_protocol_http_websocket(nc); // Poll in a loop and serve /grpcz pages for (;;) { - mg_mgr_poll(&mgr, 100); + static const int k_sleep_millis = 100; + mg_mgr_poll(&mgr, k_sleep_millis); } mg_mgr_free(&mgr); return 0; diff --git a/tools/grpcz/monitoring.proto b/tools/grpcz/monitoring.proto index ec3dfe9f74c..4d09aeb6f32 100644 --- a/tools/grpcz/monitoring.proto +++ b/tools/grpcz/monitoring.proto @@ -34,8 +34,8 @@ syntax = "proto3"; // TODO(ericgribkoff) Figure out how to manage the external Census proto // dependency. import "tools/grpcz/census.proto"; -import "tools/grpcz/empty.proto"; -import "tools/grpcz/any.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/any.proto"; package grpc.instrumentation.v1alpha; @@ -126,6 +126,6 @@ message MonitoringDataGroup { // The wrapper for custom monitoring data. message CustomMonitoringData { // can be any application specific monitoring data - Any contents = 1; + google.protobuf.Any contents = 1; } From 25e78e3213135673c368a59f159837525f118e8e Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 17 Mar 2017 11:13:09 -0700 Subject: [PATCH 10/32] removed unnecessary includes --- tools/grpcz/grpcz_client.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/grpcz/grpcz_client.cc b/tools/grpcz/grpcz_client.cc index f4611466c53..0f1432722a1 100644 --- a/tools/grpcz/grpcz_client.cc +++ b/tools/grpcz/grpcz_client.cc @@ -31,8 +31,6 @@ * */ -#include -#include #include #include From 366c9c5aaa004d04f63e3c4ff85d418e6f1f2ec5 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 20 Mar 2017 09:41:46 -0700 Subject: [PATCH 11/32] using URI query to communicate payload When using GET verb, the request payload is now communicated as a base64 encoded string in the query parameter. Deleted the code that wrote/read a custom header field (payload-bin), since caches don't consider this field when looking for a cache hit. --- src/core/lib/channel/http_client_filter.c | 41 +++++++++++++----- src/core/lib/channel/http_server_filter.c | 52 ++++++++++++++++------- 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index c031533dd86..d67b4c6e867 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -36,6 +36,7 @@ #include #include #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/security/util/b64.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -56,7 +57,6 @@ typedef struct call_data { grpc_linked_mdelem te_trailers; grpc_linked_mdelem content_type; grpc_linked_mdelem user_agent; - grpc_linked_mdelem payload_bin; grpc_metadata_batch *recv_initial_metadata; grpc_metadata_batch *recv_trailing_metadata; @@ -292,16 +292,37 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, continue_send_message(exec_ctx, elem); if (calld->send_message_blocked == false) { - /* when all the send_message data is available, then create a MDELEM and - append to headers */ - grpc_mdelem payload_bin = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_GRPC_PAYLOAD_BIN, - grpc_slice_from_copied_buffer((const char *)calld->payload_bytes, - op->send_message->length)); - error = - grpc_metadata_batch_add_tail(exec_ctx, op->send_initial_metadata, - &calld->payload_bin, payload_bin); + /* when all the send_message data is available, then modify the path + * MDELEM by appending base64 encoded query to the path */ + static const int k_url_safe = 1; + static const int k_multi_line = 0; + static char *k_query_separator = "?"; + char *strs_to_concatenate[3]; + strs_to_concatenate[0] = grpc_dump_slice( + GRPC_MDVALUE(op->send_initial_metadata->idx.named.path->md), + GPR_DUMP_ASCII); + strs_to_concatenate[1] = k_query_separator; + strs_to_concatenate[2] = grpc_base64_encode( + (const void *)calld->payload_bytes, op->send_message->length, + k_url_safe, k_multi_line); + size_t concatenated_len; + char *path_with_query = gpr_strjoin((const char **)strs_to_concatenate, + 3, &concatenated_len); + gpr_log(GPR_DEBUG, "Path with query: %s\n", path_with_query); + gpr_free(strs_to_concatenate[0]); + gpr_free(strs_to_concatenate[2]); + + /* substitute previous path with the new path */ + grpc_mdelem mdelem_path_and_query = grpc_mdelem_from_slices( + exec_ctx, GRPC_MDSTR_PATH, + grpc_slice_from_copied_buffer((const char *)path_with_query, + concatenated_len)); + gpr_free(path_with_query); + grpc_metadata_batch *b = op->send_initial_metadata; + error = grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, + mdelem_path_and_query); if (error != GRPC_ERROR_NONE) return error; + calld->on_complete = op->on_complete; op->on_complete = &calld->hc_on_complete; op->send_message = NULL; diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index fb70de8e96c..d7a4ba31b35 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -37,6 +37,7 @@ #include #include #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/security/util/b64.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -51,8 +52,8 @@ typedef struct call_data { grpc_linked_mdelem status; grpc_linked_mdelem content_type; - /* did this request come with payload-bin */ - bool seen_payload_bin; + /* did this request come with path query containing request payload */ + bool seen_path_with_query; /* flag to ensure payload_bin is delivered only once */ bool payload_bin_delivered; @@ -61,7 +62,7 @@ typedef struct call_data { bool *recv_cacheable_request; /** Closure to call when finished with the hs_on_recv hook */ grpc_closure *on_done_recv; - /** Closure to call when we retrieve read message from the payload-bin header + /** Closure to call when we retrieve read message from the path URI */ grpc_closure *recv_message_ready; grpc_closure *on_complete; @@ -196,6 +197,35 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, add_error(error_name, &error, grpc_error_set_str(GRPC_ERROR_CREATE("Missing header"), GRPC_ERROR_STR_KEY, ":path")); + } else if (*calld->recv_cacheable_request == true) { + /* We have a cacheable request made with GET verb. The path contains the + * query parameter which is base64 encoded request payload. */ + char *path = + grpc_dump_slice(GRPC_MDVALUE(b->idx.named.path->md), GPR_DUMP_ASCII); + static const char *QUERY_SEPARATOR = "?"; + char **query_parts; + size_t num_query_parts; + gpr_string_split(path, QUERY_SEPARATOR, &query_parts, &num_query_parts); + GPR_ASSERT(num_query_parts == 2); + /* substitute path metadata with just the path (not query) */ + grpc_mdelem mdelem_path_without_query = grpc_mdelem_from_slices( + exec_ctx, GRPC_MDSTR_PATH, + grpc_slice_from_copied_buffer((const char *)query_parts[0], + strlen(query_parts[0]))); + grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, + mdelem_path_without_query); + gpr_free(query_parts[0]); + + /* decode query into payload and add it to the slice buffer to be returned + * */ + static const int k_url_safe = 1; + grpc_slice_buffer_add( + &calld->read_slice_buffer, + grpc_base64_decode(exec_ctx, (const char *)query_parts[1], k_url_safe)); + grpc_slice_buffer_stream_init(&calld->read_stream, + &calld->read_slice_buffer, 0); + gpr_free(query_parts[1]); + calld->seen_path_with_query = true; } if (b->idx.named.host != NULL && b->idx.named.authority == NULL) { @@ -217,16 +247,6 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, GRPC_ERROR_STR_KEY, ":authority")); } - if (b->idx.named.grpc_payload_bin != NULL) { - calld->seen_payload_bin = true; - grpc_slice_buffer_add(&calld->read_slice_buffer, - grpc_slice_ref_internal( - GRPC_MDVALUE(b->idx.named.grpc_payload_bin->md))); - grpc_slice_buffer_stream_init(&calld->read_stream, - &calld->read_slice_buffer, 0); - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_payload_bin); - } - return error; } @@ -247,8 +267,8 @@ static void hs_on_complete(grpc_exec_ctx *exec_ctx, void *user_data, grpc_error *err) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - /* Call recv_message_ready if we got the payload via the header field */ - if (calld->seen_payload_bin && calld->recv_message_ready != NULL) { + /* Call recv_message_ready if we got the payload via the path field */ + if (calld->seen_path_with_query && calld->recv_message_ready != NULL) { *calld->pp_recv_message = calld->payload_bin_delivered ? NULL : (grpc_byte_stream *)&calld->read_stream; @@ -263,7 +283,7 @@ static void hs_recv_message_ready(grpc_exec_ctx *exec_ctx, void *user_data, grpc_error *err) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - if (calld->seen_payload_bin) { + if (calld->seen_path_with_query) { /* do nothing. This is probably a GET request, and payload will be returned in hs_on_complete callback. */ } else { From b239da9ab37042ceaf8c5183c0f7511aab352c63 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 20 Mar 2017 13:49:49 -0700 Subject: [PATCH 12/32] removed multiple allocs, refactored b64 encoder Added new api to b64.h for directly encoding to memory. Using slice_sub instead of string operations for separating path from query. --- src/core/lib/channel/http_client_filter.c | 58 +++++++++++++++-------- src/core/lib/channel/http_server_filter.c | 33 +++++++------ src/core/lib/security/util/b64.c | 25 ++++++++-- src/core/lib/security/util/b64.h | 11 +++++ 4 files changed, 86 insertions(+), 41 deletions(-) diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index d67b4c6e867..37bc07da895 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -296,28 +296,44 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, * MDELEM by appending base64 encoded query to the path */ static const int k_url_safe = 1; static const int k_multi_line = 0; - static char *k_query_separator = "?"; - char *strs_to_concatenate[3]; - strs_to_concatenate[0] = grpc_dump_slice( - GRPC_MDVALUE(op->send_initial_metadata->idx.named.path->md), - GPR_DUMP_ASCII); - strs_to_concatenate[1] = k_query_separator; - strs_to_concatenate[2] = grpc_base64_encode( - (const void *)calld->payload_bytes, op->send_message->length, - k_url_safe, k_multi_line); - size_t concatenated_len; - char *path_with_query = gpr_strjoin((const char **)strs_to_concatenate, - 3, &concatenated_len); - gpr_log(GPR_DEBUG, "Path with query: %s\n", path_with_query); - gpr_free(strs_to_concatenate[0]); - gpr_free(strs_to_concatenate[2]); - - /* substitute previous path with the new path */ + static const char *k_query_separator = "?"; + + grpc_slice path_slice = + GRPC_MDVALUE(op->send_initial_metadata->idx.named.path->md); + /* sum up individual component's lengths and allocate enough memory to + * hold combined path+query */ + size_t estimated_len = GRPC_SLICE_LENGTH(path_slice); + estimated_len += strlen(k_query_separator); + estimated_len += grpc_base64_estimate_encoded_size( + op->send_message->length, k_url_safe, k_multi_line); + estimated_len += 1; /* for the trailing 0 */ + grpc_slice path_with_query_slice = grpc_slice_malloc(estimated_len); + + /* memcopy individual pieces into this slice */ + char *write_ptr = (char *)GRPC_SLICE_START_PTR(path_with_query_slice); + + char *original_path = (char *)GRPC_SLICE_START_PTR(path_slice); + memcpy(write_ptr, original_path, GRPC_SLICE_LENGTH(path_slice)); + write_ptr += (int)GRPC_SLICE_LENGTH(path_slice); + + memcpy(write_ptr, k_query_separator, strlen(k_query_separator)); + write_ptr += strlen(k_query_separator); + + grpc_base64_encode_core(write_ptr, calld->payload_bytes, + op->send_message->length, k_url_safe, + k_multi_line); + + /* remove trailing unused memory and add trailing 0 to terminate string + */ + char *t = (char *)GRPC_SLICE_START_PTR(path_with_query_slice); + size_t path_length = strlen(t) + 1; + *(t + path_length) = 0; + path_with_query_slice = + grpc_slice_sub(path_with_query_slice, 0, path_length); + + /* substitute previous path with the new path+query */ grpc_mdelem mdelem_path_and_query = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_PATH, - grpc_slice_from_copied_buffer((const char *)path_with_query, - concatenated_len)); - gpr_free(path_with_query); + exec_ctx, GRPC_MDSTR_PATH, path_with_query_slice); grpc_metadata_batch *b = op->send_initial_metadata; error = grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, mdelem_path_and_query); diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index d7a4ba31b35..31149aa1169 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -200,31 +200,34 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, } else if (*calld->recv_cacheable_request == true) { /* We have a cacheable request made with GET verb. The path contains the * query parameter which is base64 encoded request payload. */ - char *path = - grpc_dump_slice(GRPC_MDVALUE(b->idx.named.path->md), GPR_DUMP_ASCII); - static const char *QUERY_SEPARATOR = "?"; - char **query_parts; - size_t num_query_parts; - gpr_string_split(path, QUERY_SEPARATOR, &query_parts, &num_query_parts); - GPR_ASSERT(num_query_parts == 2); + static const char *k_query_separator = "?"; + grpc_slice path_slice = GRPC_MDVALUE(b->idx.named.path->md); + char *path_ptr = (char *)GRPC_SLICE_START_PTR(path_slice); + size_t path_length = GRPC_SLICE_LENGTH(path_slice); + /* offset of the character '?' */ + size_t offset = 0; + for (offset = 0; *path_ptr != k_query_separator[0] && offset < path_length; + path_ptr++, offset++) + ; + grpc_slice query_slice = + grpc_slice_sub(path_slice, offset + 1, path_length); + /* substitute path metadata with just the path (not query) */ grpc_mdelem mdelem_path_without_query = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_PATH, - grpc_slice_from_copied_buffer((const char *)query_parts[0], - strlen(query_parts[0]))); + exec_ctx, GRPC_MDSTR_PATH, grpc_slice_sub(path_slice, 0, offset)); + grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, mdelem_path_without_query); - gpr_free(query_parts[0]); - /* decode query into payload and add it to the slice buffer to be returned - * */ + /* decode payload from query and add to the slice buffer to be returned */ static const int k_url_safe = 1; grpc_slice_buffer_add( &calld->read_slice_buffer, - grpc_base64_decode(exec_ctx, (const char *)query_parts[1], k_url_safe)); + grpc_base64_decode(exec_ctx, + (const char *)GRPC_SLICE_START_PTR(query_slice), + k_url_safe)); grpc_slice_buffer_stream_init(&calld->read_stream, &calld->read_slice_buffer, 0); - gpr_free(query_parts[1]); calld->seen_path_with_query = true; } diff --git a/src/core/lib/security/util/b64.c b/src/core/lib/security/util/b64.c index 09c82131316..a84e42b4f6e 100644 --- a/src/core/lib/security/util/b64.c +++ b/src/core/lib/security/util/b64.c @@ -71,15 +71,31 @@ static const char base64_url_safe_chars[] = char *grpc_base64_encode(const void *vdata, size_t data_size, int url_safe, int multiline) { - const unsigned char *data = vdata; - const char *base64_chars = - url_safe ? base64_url_safe_chars : base64_url_unsafe_chars; + size_t result_projected_size = + grpc_base64_estimate_encoded_size(data_size, url_safe, multiline); + char *result = gpr_malloc(result_projected_size); + grpc_base64_encode_core(result, vdata, data_size, url_safe, multiline); + return result; +} + +size_t grpc_base64_estimate_encoded_size(size_t data_size, int url_safe, + int multiline) { size_t result_projected_size = 4 * ((data_size + 3) / 3) + 2 * (multiline ? (data_size / (3 * GRPC_BASE64_MULTILINE_NUM_BLOCKS)) : 0) + 1; - char *result = gpr_malloc(result_projected_size); + return result_projected_size; +} + +void grpc_base64_encode_core(char *result, const void *vdata, size_t data_size, + int url_safe, int multiline) { + const unsigned char *data = vdata; + const char *base64_chars = + url_safe ? base64_url_safe_chars : base64_url_unsafe_chars; + size_t result_projected_size = + grpc_base64_estimate_encoded_size(data_size, url_safe, multiline); + char *current = result; size_t num_blocks = 0; size_t i = 0; @@ -119,7 +135,6 @@ char *grpc_base64_encode(const void *vdata, size_t data_size, int url_safe, GPR_ASSERT(current >= result); GPR_ASSERT((uintptr_t)(current - result) < result_projected_size); result[current - result] = '\0'; - return result; } grpc_slice grpc_base64_decode(grpc_exec_ctx *exec_ctx, const char *b64, diff --git a/src/core/lib/security/util/b64.h b/src/core/lib/security/util/b64.h index d42a136f61a..e84c50271e3 100644 --- a/src/core/lib/security/util/b64.h +++ b/src/core/lib/security/util/b64.h @@ -41,6 +41,17 @@ char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, int multiline); +/* estimate the upper bound on size of base64 encoded data. The actual size + * is guaranteed to be less than or equal to the size returned here. */ +size_t grpc_base64_estimate_encoded_size(size_t data_size, int url_safe, + int multiline); + +/* Encodes data using base64 and write it to memory pointed to by result. It is + * the caller's responsiblity to allocate enough memory in |result| to fit the + * encoded data. */ +void grpc_base64_encode_core(char *result, const void *vdata, size_t data_size, + int url_safe, int multiline); + /* Decodes data according to the base64 specification. Returns an empty slice in case of failure. */ grpc_slice grpc_base64_decode(grpc_exec_ctx *exec_ctx, const char *b64, From 2c19723e66d3f704e17454da62c680c7069b94d3 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 20 Mar 2017 16:00:15 -0700 Subject: [PATCH 13/32] updated monitoring.proto Also changed the name of stats attr to 'data-stats' to be more html5 compliant. --- tools/grpcz/grpcz_client.cc | 4 +- tools/grpcz/monitoring.proto | 85 +++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/tools/grpcz/grpcz_client.cc b/tools/grpcz/grpcz_client.cc index 0f1432722a1..47eec8dfc38 100644 --- a/tools/grpcz/grpcz_client.cc +++ b/tools/grpcz/grpcz_client.cc @@ -65,14 +65,14 @@ static const std::string static_html_header = table { border-collapse: collapse; width: 100%; } \ table, td, th { border: 1px solid black; } \ \ -\

GRPCZ Statistics

\