From 44de6a1701d8e9ce5617fe93d816e0ac45172f0c Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 10 Feb 2016 09:58:13 -0800 Subject: [PATCH 01/17] Added binary log proto --- src/proto/grpc/binary_log/v1alpha/log.proto | 84 +++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 src/proto/grpc/binary_log/v1alpha/log.proto diff --git a/src/proto/grpc/binary_log/v1alpha/log.proto b/src/proto/grpc/binary_log/v1alpha/log.proto new file mode 100644 index 00000000000..43277bc4a5c --- /dev/null +++ b/src/proto/grpc/binary_log/v1alpha/log.proto @@ -0,0 +1,84 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +import "google/prootbuf/timestamp.proto" + +package grpc.binary_log.v1alpha; + +enum Direction { + SERVER_SEND; + SERVER_RECV; + CLIENT_SEND; + CLIENT_RECV; +} + +message KeyValuePair { + string key; + string value; +} + +message Metadata { + uint32 rpc_id = 1; + google.protobuf.Timestamp timestamp = 2; + Direction direction = 3; + repeated KeyValuePair metadata = 4; + + message ClientInitialMetadata { + string method_name = 1; + google.protobuf.Timestamp timestamp = 2; + } + + message UserData { + } + + message ServerInitialMetadata { + } + + message ServerStatus { + uint32 code = 1; + string details = 2; + } + + oneof kind { + ClientInitialMetadata client_initial_metadata = 5; + UserData user_data = 6; + ServerInitialMetadata server_initial_metadata = 7; + ServerStatus server_status = 8; + } +} + +message Message { + uint32 rpc_id = 1; + uint32 sequence_number = 2; + Direction direction = 3; + uint64 length; + bytes data; +} \ No newline at end of file From f8c6545ae838be5b7704a8147668c19d6adf676f Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 10 Feb 2016 13:55:08 -0800 Subject: [PATCH 02/17] Update binary log proto, add document explaining it --- doc/binary-logging.md | 52 +++++++++++++++++++++ src/proto/grpc/binary_log/v1alpha/log.proto | 34 ++++++++++++-- 2 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 doc/binary-logging.md diff --git a/doc/binary-logging.md b/doc/binary-logging.md new file mode 100644 index 00000000000..84fd9914116 --- /dev/null +++ b/doc/binary-logging.md @@ -0,0 +1,52 @@ +# Binary Logging + +## Format + +The log format is described in [this proto file](src/proto/grpc/binary_log/v1alpha/log.proto). It is intended that multiple parts of the call will be logged in separate files, and then correlated by analysis tools using the rpc\_id. + +## API + +The binary logger will be a separate library from gRPC, in each language that we support. The user will need to explicitly call into the library to generate logs. The following API is an example of what it will approximately look like in C++: + +```c++ +// The context provides the method_name, deadline, peer, and metadata contents. +// direction = CLIENT_SEND +LogRequestHeaders(ClientContext context); +// direction = SERVER_RECV +LogRequestHeaders(ServerContext context); + +// The context provides the metadata contents +// direction = CLIENT_RECV +LogResponseHeaders(ClientContext context); +// direction = SERVER_SEND +LogResponseHeaders(ServerContext context); + +// The context provides the metadata contents +// direction = CLIENT_RECV +LogStatus(ClientContext context, grpc_status_code code, string details); +// direction = SERVER_SEND +LogStatus(ServerContext context, grpc_status_code code, string details); + +// The context provides the user data contents +// direction = CLIENT_SEND +LogUserData(ClientContext context); +// direction = SERVER_SEND +LogUserData(ServerContext context); + +// direction = CLIENT_SEND +LogRequestMessage(ClientContext context, uint32_t length, T message); +// direction = SERVER_RECV +LogRequestMessage(ServerContext context, uint32_t length, T message); +// direction = CLIENT_RECV +LogResponseMessage(ClientContext context, uint32_t length, T message); +// direction = SERVER_SEND +LogResponseMessage(ServerContext context, uint32_t length, T message); +``` + +In all of those cases, the `rpc_id` is provided by the context, and each combination of method and context argument type implies a single direction, as noted in the comments. + +For the message log functions, the `length` argument indicates the length of the complete message, and the `message` argument may be only part of the complete message, stripped of sensitive material and/or shortened for efficiency. + +## Language differences + +In other languages, more or less data will need to be passed explicitly as separate arguments. In some languages, for example, the metadata will be separate from the context-like object and will need to be passed as a separate argument. diff --git a/src/proto/grpc/binary_log/v1alpha/log.proto b/src/proto/grpc/binary_log/v1alpha/log.proto index 43277bc4a5c..8190fa0f306 100644 --- a/src/proto/grpc/binary_log/v1alpha/log.proto +++ b/src/proto/grpc/binary_log/v1alpha/log.proto @@ -45,25 +45,41 @@ message KeyValuePair { string value; } +// Any sort of metadata that may be sent in either direction during a call message Metadata { - uint32 rpc_id = 1; + // Cryptographically unique identifier, generated on the client and sent + // to the server. + uint64 rpc_id = 1; + // Timestamp of logging the metadata google.protobuf.Timestamp timestamp = 2; Direction direction = 3; + // The actual metadata that is being sent repeated KeyValuePair metadata = 4; + // Initial metadata sent by the client to initiate a request message ClientInitialMetadata { + // The full method name that is being called string method_name = 1; - google.protobuf.Timestamp timestamp = 2; + // The call's deadline + google.protobuf.Timestamp deadline = 2; + // The address of the connected peer + string peer = 3; } + // Arbitrary key/value pairs specified by the user that are not sent over + // the network but are nonetheless useful to log message UserData { } + // Initial metadata response sent by the server after accepting the request message ServerInitialMetadata { } + // Status sent by the server when closing the call on the server side message ServerStatus { + // The status code uint32 code = 1; + // The status details string details = 2; } @@ -75,10 +91,18 @@ message Metadata { } } +// A message that is sent during a call message Message { + // Cryptographically unique identifier, generated on the client and sent + // to the server. uint32 rpc_id = 1; - uint32 sequence_number = 2; + // The sequence number of the message. Messages sent by the client and by the + // server should have independently incrementing sequence numbers. + uint64 sequence_number = 2; Direction direction = 3; - uint64 length; - bytes data; + // The length of the complete message. + uint32 length = 4; + // The contents of the message. May be a prefix instead of the complete + // message. + bytes data = 5; } \ No newline at end of file From 18c4a532d564e1e38c458c2b400160a2a0a38f68 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 10 Feb 2016 13:57:38 -0800 Subject: [PATCH 03/17] Fixed inconsistency in a type in the binary log proto --- src/proto/grpc/binary_log/v1alpha/log.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/proto/grpc/binary_log/v1alpha/log.proto b/src/proto/grpc/binary_log/v1alpha/log.proto index 8190fa0f306..c338ea760b9 100644 --- a/src/proto/grpc/binary_log/v1alpha/log.proto +++ b/src/proto/grpc/binary_log/v1alpha/log.proto @@ -95,10 +95,10 @@ message Metadata { message Message { // Cryptographically unique identifier, generated on the client and sent // to the server. - uint32 rpc_id = 1; + uint64 rpc_id = 1; // The sequence number of the message. Messages sent by the client and by the // server should have independently incrementing sequence numbers. - uint64 sequence_number = 2; + uint32 sequence_number = 2; Direction direction = 3; // The length of the complete message. uint32 length = 4; From 3c49ade48500758abcd0e701a0c7c674356e889e Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 16 Feb 2016 14:11:56 -0800 Subject: [PATCH 04/17] Clarify API section of binary logging doc --- doc/binary-logging.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/binary-logging.md b/doc/binary-logging.md index 84fd9914116..69020d98285 100644 --- a/doc/binary-logging.md +++ b/doc/binary-logging.md @@ -6,7 +6,14 @@ The log format is described in [this proto file](src/proto/grpc/binary_log/v1alp ## API -The binary logger will be a separate library from gRPC, in each language that we support. The user will need to explicitly call into the library to generate logs. The following API is an example of what it will approximately look like in C++: +The binary logger will be a separate library from gRPC, in each language that we support. The user will need to explicitly call into the library to generate logs. The library will provide the ability to log sending or receiving, as relevant, the following on both the client and the server: + + - Initial metadata + - Messages + - Status with trailing metadata from the server + - Additional key/value pairs that are associated with a call but not sent over the wire + +The following is an example of what such an API could look like in C++: ```c++ // The context provides the method_name, deadline, peer, and metadata contents. From c2ec738f6937ccffbfa0887ba238f5dc1df3304f Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 30 Mar 2016 15:27:56 -0700 Subject: [PATCH 05/17] Fixed typo in import statement --- src/proto/grpc/binary_log/v1alpha/log.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/proto/grpc/binary_log/v1alpha/log.proto b/src/proto/grpc/binary_log/v1alpha/log.proto index c338ea760b9..6cc473be74e 100644 --- a/src/proto/grpc/binary_log/v1alpha/log.proto +++ b/src/proto/grpc/binary_log/v1alpha/log.proto @@ -29,7 +29,7 @@ syntax = "proto3"; -import "google/prootbuf/timestamp.proto" +import "google/protobuf/timestamp.proto" package grpc.binary_log.v1alpha; From 6c621069f0fbcfbcd614c6dcff2389db1eb10d60 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 1 Apr 2016 09:33:55 -0700 Subject: [PATCH 06/17] Remove some debug logging from 0.13-branch node --- src/node/src/credentials.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/node/src/credentials.js b/src/node/src/credentials.js index 1d73723cc06..97c4bd73ac4 100644 --- a/src/node/src/credentials.js +++ b/src/node/src/credentials.js @@ -118,7 +118,6 @@ exports.createFromMetadataGenerator = function(metadata_generator) { exports.createFromGoogleCredential = function(google_credential) { return exports.createFromMetadataGenerator(function(auth_context, callback) { var service_url = auth_context.service_url; - console.log('Service URL:', service_url); google_credential.getRequestMetadata(service_url, function(err, header) { if (err) { console.log('Auth error:', err); @@ -127,7 +126,6 @@ exports.createFromGoogleCredential = function(google_credential) { } var metadata = new Metadata(); metadata.add('authorization', header.Authorization); - console.log(header.Authorization); callback(null, metadata); }); }); From 70f50b13de40e2f0b4815a0c640a116043c2cdeb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 1 Apr 2016 10:13:08 -0700 Subject: [PATCH 07/17] upgrade helloworld and routeguide examples to 0.13.1 --- examples/csharp/helloworld/.nuget/packages.config | 2 +- examples/csharp/helloworld/Greeter/Greeter.csproj | 10 +++++----- .../csharp/helloworld/Greeter/packages.config | 6 +++--- .../helloworld/GreeterClient/GreeterClient.csproj | 10 +++++----- .../helloworld/GreeterClient/packages.config | 6 +++--- .../helloworld/GreeterServer/GreeterServer.csproj | 10 +++++----- .../helloworld/GreeterServer/packages.config | 6 +++--- .../csharp/route_guide/.nuget/packages.config | 2 +- .../route_guide/RouteGuide/RouteGuide.csproj | 15 ++++++++------- .../csharp/route_guide/RouteGuide/packages.config | 8 ++++---- .../RouteGuideClient/RouteGuideClient.csproj | 15 ++++++++------- .../route_guide/RouteGuideClient/packages.config | 8 ++++---- .../RouteGuideServer/RouteGuideServer.csproj | 15 ++++++++------- .../route_guide/RouteGuideServer/packages.config | 8 ++++---- 14 files changed, 62 insertions(+), 59 deletions(-) diff --git a/examples/csharp/helloworld/.nuget/packages.config b/examples/csharp/helloworld/.nuget/packages.config index fb778311d1f..0f89a66e25f 100644 --- a/examples/csharp/helloworld/.nuget/packages.config +++ b/examples/csharp/helloworld/.nuget/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/examples/csharp/helloworld/Greeter/Greeter.csproj b/examples/csharp/helloworld/Greeter/Greeter.csproj index a25ad5f7910..89e9f16b563 100644 --- a/examples/csharp/helloworld/Greeter/Greeter.csproj +++ b/examples/csharp/helloworld/Greeter/Greeter.csproj @@ -10,7 +10,7 @@ Greeter Greeter v4.5 - 39f4a691 + 96275748 true @@ -35,9 +35,9 @@ False ..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll - + False - ..\packages\Grpc.Core.0.13.0\lib\net45\Grpc.Core.dll + ..\packages\Grpc.Core.0.13.1\lib\net45\Grpc.Core.dll @@ -56,11 +56,11 @@ - + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + \ No newline at end of file diff --git a/examples/csharp/helloworld/Greeter/packages.config b/examples/csharp/helloworld/Greeter/packages.config index abe9ad08507..7d24440b123 100644 --- a/examples/csharp/helloworld/Greeter/packages.config +++ b/examples/csharp/helloworld/Greeter/packages.config @@ -1,8 +1,8 @@  - - - + + + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterClient/GreeterClient.csproj b/examples/csharp/helloworld/GreeterClient/GreeterClient.csproj index a71cfeeef33..e10a6058df1 100644 --- a/examples/csharp/helloworld/GreeterClient/GreeterClient.csproj +++ b/examples/csharp/helloworld/GreeterClient/GreeterClient.csproj @@ -10,7 +10,7 @@ GreeterClient GreeterClient v4.5 - dcebbc77 + d94f6f5f true @@ -35,9 +35,9 @@ False ..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll - + False - ..\packages\Grpc.Core.0.13.0\lib\net45\Grpc.Core.dll + ..\packages\Grpc.Core.0.13.1\lib\net45\Grpc.Core.dll @@ -59,11 +59,11 @@ - + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterClient/packages.config b/examples/csharp/helloworld/GreeterClient/packages.config index abe9ad08507..7d24440b123 100644 --- a/examples/csharp/helloworld/GreeterClient/packages.config +++ b/examples/csharp/helloworld/GreeterClient/packages.config @@ -1,8 +1,8 @@  - - - + + + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterServer/GreeterServer.csproj b/examples/csharp/helloworld/GreeterServer/GreeterServer.csproj index 34eea8c246d..6c70f75087d 100644 --- a/examples/csharp/helloworld/GreeterServer/GreeterServer.csproj +++ b/examples/csharp/helloworld/GreeterServer/GreeterServer.csproj @@ -10,7 +10,7 @@ GreeterServer GreeterServer v4.5 - 2ea5dfd0 + 6f89e9f2 true @@ -35,9 +35,9 @@ False ..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll - + False - ..\packages\Grpc.Core.0.13.0\lib\net45\Grpc.Core.dll + ..\packages\Grpc.Core.0.13.1\lib\net45\Grpc.Core.dll @@ -59,11 +59,11 @@ - + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterServer/packages.config b/examples/csharp/helloworld/GreeterServer/packages.config index abe9ad08507..7d24440b123 100644 --- a/examples/csharp/helloworld/GreeterServer/packages.config +++ b/examples/csharp/helloworld/GreeterServer/packages.config @@ -1,8 +1,8 @@  - - - + + + \ No newline at end of file diff --git a/examples/csharp/route_guide/.nuget/packages.config b/examples/csharp/route_guide/.nuget/packages.config index fb778311d1f..0f89a66e25f 100644 --- a/examples/csharp/route_guide/.nuget/packages.config +++ b/examples/csharp/route_guide/.nuget/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj b/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj index 1fbf1ce1831..8ff115d252f 100644 --- a/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj +++ b/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj @@ -11,7 +11,7 @@ RouteGuide v4.5 512 - 5b6d924a + e1e648e7 true @@ -35,9 +35,9 @@ False ..\packages\Google.Protobuf.3.0.0-beta2\lib\portable-net45+netcore45+wpa81+wp8\Google.Protobuf.dll - + False - ..\packages\Grpc.Core.0.13.0\lib\net45\Grpc.Core.dll + ..\packages\Grpc.Core.0.13.1\lib\net45\Grpc.Core.dll False @@ -45,8 +45,9 @@ - - ..\packages\Ix-Async.1.2.3\lib\net45\System.Interactive.Async.dll + + False + ..\packages\Ix-Async.1.2.5\lib\net45\System.Interactive.Async.dll @@ -67,12 +68,12 @@ - + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - +