diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000000..700c61cceae --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,30 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Mocha Tests", + "cwd": "${workspaceRoot}", + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha", + "windows": { + "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha.cmd" + }, + "runtimeArgs": [ + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceRoot}/src/node/test" + ], + "internalConsoleOptions": "openOnSessionStart" + }, + { + "type": "node", + "request": "attach", + "name": "Attach to Process", + "port": 5858 + } + ] +} diff --git a/BUILD b/BUILD index 1cf4782eaf7..d6598487fba 100644 --- a/BUILD +++ b/BUILD @@ -511,6 +511,7 @@ grpc_cc_library( "src/core/lib/json/json_reader.c", "src/core/lib/json/json_string.c", "src/core/lib/json/json_writer.c", + "src/core/lib/security/util/b64.c", "src/core/lib/slice/percent_encoding.c", "src/core/lib/slice/slice.c", "src/core/lib/slice/slice_buffer.c", @@ -629,6 +630,7 @@ grpc_cc_library( "src/core/lib/json/json_common.h", "src/core/lib/json/json_reader.h", "src/core/lib/json/json_writer.h", + "src/core/lib/security/util/b64.h", "src/core/lib/slice/percent_encoding.h", "src/core/lib/slice/slice_hash_table.h", "src/core/lib/slice/slice_internal.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index dac4095a545..f54e78b560d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -993,6 +993,7 @@ add_library(grpc src/core/lib/json/json_reader.c src/core/lib/json/json_string.c src/core/lib/json/json_writer.c + src/core/lib/security/util/b64.c src/core/lib/slice/percent_encoding.c src/core/lib/slice/slice.c src/core/lib/slice/slice_buffer.c @@ -1075,7 +1076,6 @@ add_library(grpc src/core/lib/security/transport/security_handshaker.c src/core/lib/security/transport/server_auth_filter.c src/core/lib/security/transport/tsi_error.c - src/core/lib/security/util/b64.c src/core/lib/security/util/json_util.c src/core/lib/surface/init_secure.c src/core/lib/tsi/fake_transport_security.c @@ -1314,6 +1314,7 @@ add_library(grpc_cronet src/core/lib/json/json_reader.c src/core/lib/json/json_string.c src/core/lib/json/json_writer.c + src/core/lib/security/util/b64.c src/core/lib/slice/percent_encoding.c src/core/lib/slice/slice.c src/core/lib/slice/slice_buffer.c @@ -1419,7 +1420,6 @@ add_library(grpc_cronet src/core/lib/security/transport/security_handshaker.c src/core/lib/security/transport/server_auth_filter.c src/core/lib/security/transport/tsi_error.c - src/core/lib/security/util/b64.c src/core/lib/security/util/json_util.c src/core/lib/surface/init_secure.c src/core/lib/tsi/fake_transport_security.c @@ -1622,6 +1622,7 @@ add_library(grpc_test_util src/core/lib/json/json_reader.c src/core/lib/json/json_string.c src/core/lib/json/json_writer.c + src/core/lib/security/util/b64.c src/core/lib/slice/percent_encoding.c src/core/lib/slice/slice.c src/core/lib/slice/slice_buffer.c @@ -1881,6 +1882,7 @@ add_library(grpc_unsecure src/core/lib/json/json_reader.c src/core/lib/json/json_string.c src/core/lib/json/json_writer.c + src/core/lib/security/util/b64.c src/core/lib/slice/percent_encoding.c src/core/lib/slice/slice.c src/core/lib/slice/slice_buffer.c @@ -2511,6 +2513,7 @@ add_library(grpc++_cronet src/core/lib/json/json_reader.c src/core/lib/json/json_string.c src/core/lib/json/json_writer.c + src/core/lib/security/util/b64.c src/core/lib/slice/percent_encoding.c src/core/lib/slice/slice.c src/core/lib/slice/slice_buffer.c @@ -11360,7 +11363,6 @@ add_executable(stress_test test/cpp/interop/interop_client.cc test/cpp/interop/stress_interop_client.cc test/cpp/interop/stress_test.cc - test/cpp/util/create_test_channel.cc test/cpp/util/metrics_server.cc third_party/googletest/src/gtest-all.cc ) diff --git a/Makefile b/Makefile index 4985907f5c1..bfc43aab28e 100644 --- a/Makefile +++ b/Makefile @@ -2887,6 +2887,7 @@ LIBGRPC_SRC = \ src/core/lib/json/json_reader.c \ src/core/lib/json/json_string.c \ src/core/lib/json/json_writer.c \ + src/core/lib/security/util/b64.c \ src/core/lib/slice/percent_encoding.c \ src/core/lib/slice/slice.c \ src/core/lib/slice/slice_buffer.c \ @@ -2969,7 +2970,6 @@ LIBGRPC_SRC = \ src/core/lib/security/transport/security_handshaker.c \ src/core/lib/security/transport/server_auth_filter.c \ src/core/lib/security/transport/tsi_error.c \ - src/core/lib/security/util/b64.c \ src/core/lib/security/util/json_util.c \ src/core/lib/surface/init_secure.c \ src/core/lib/tsi/fake_transport_security.c \ @@ -3206,6 +3206,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/json/json_reader.c \ src/core/lib/json/json_string.c \ src/core/lib/json/json_writer.c \ + src/core/lib/security/util/b64.c \ src/core/lib/slice/percent_encoding.c \ src/core/lib/slice/slice.c \ src/core/lib/slice/slice_buffer.c \ @@ -3311,7 +3312,6 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/security/transport/security_handshaker.c \ src/core/lib/security/transport/server_auth_filter.c \ src/core/lib/security/transport/tsi_error.c \ - src/core/lib/security/util/b64.c \ src/core/lib/security/util/json_util.c \ src/core/lib/surface/init_secure.c \ src/core/lib/tsi/fake_transport_security.c \ @@ -3513,6 +3513,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/json/json_reader.c \ src/core/lib/json/json_string.c \ src/core/lib/json/json_writer.c \ + src/core/lib/security/util/b64.c \ src/core/lib/slice/percent_encoding.c \ src/core/lib/slice/slice.c \ src/core/lib/slice/slice_buffer.c \ @@ -3744,6 +3745,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/json/json_reader.c \ src/core/lib/json/json_string.c \ src/core/lib/json/json_writer.c \ + src/core/lib/security/util/b64.c \ src/core/lib/slice/percent_encoding.c \ src/core/lib/slice/slice.c \ src/core/lib/slice/slice_buffer.c \ @@ -4360,6 +4362,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/json/json_reader.c \ src/core/lib/json/json_string.c \ src/core/lib/json/json_writer.c \ + src/core/lib/security/util/b64.c \ src/core/lib/slice/percent_encoding.c \ src/core/lib/slice/slice.c \ src/core/lib/slice/slice_buffer.c \ @@ -15744,7 +15747,6 @@ STRESS_TEST_SRC = \ test/cpp/interop/interop_client.cc \ test/cpp/interop/stress_interop_client.cc \ test/cpp/interop/stress_test.cc \ - test/cpp/util/create_test_channel.cc \ test/cpp/util/metrics_server.cc \ STRESS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(STRESS_TEST_SRC)))) @@ -15790,8 +15792,6 @@ $(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_interop_client.o: $(LIBDIR)/$(CONFI $(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a -$(OBJDIR)/$(CONFIG)/test/cpp/util/create_test_channel.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a - $(OBJDIR)/$(CONFIG)/test/cpp/util/metrics_server.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a deps_stress_test: $(STRESS_TEST_OBJS:.o=.dep) @@ -15804,7 +15804,6 @@ endif $(OBJDIR)/$(CONFIG)/test/cpp/interop/interop_client.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_interop_client.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_test.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/create_test_channel.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/util/metrics_server.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc @@ -18806,7 +18805,6 @@ src/core/lib/security/transport/security_connector.c: $(OPENSSL_DEP) src/core/lib/security/transport/security_handshaker.c: $(OPENSSL_DEP) src/core/lib/security/transport/server_auth_filter.c: $(OPENSSL_DEP) src/core/lib/security/transport/tsi_error.c: $(OPENSSL_DEP) -src/core/lib/security/util/b64.c: $(OPENSSL_DEP) src/core/lib/security/util/json_util.c: $(OPENSSL_DEP) src/core/lib/surface/init_secure.c: $(OPENSSL_DEP) src/core/lib/tsi/fake_transport_security.c: $(OPENSSL_DEP) diff --git a/WORKSPACE b/WORKSPACE index 54d27f06a9e..da63c5fea80 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -72,6 +72,12 @@ local_repository( path = "third_party/gflags", ) +git_repository( + name = "mongoose_repo", + commit = "4120a97945b41195a6223a600dae8e3b19bed19e", + remote = "https://github.com/makdharma/mongoose.git" +) + new_local_repository( name = "submodule_cares", path = "third_party/cares", diff --git a/binding.gyp b/binding.gyp index 26904ae2f7b..b91f5f61e21 100644 --- a/binding.gyp +++ b/binding.gyp @@ -700,6 +700,7 @@ 'src/core/lib/json/json_reader.c', 'src/core/lib/json/json_string.c', 'src/core/lib/json/json_writer.c', + 'src/core/lib/security/util/b64.c', 'src/core/lib/slice/percent_encoding.c', 'src/core/lib/slice/slice.c', 'src/core/lib/slice/slice_buffer.c', @@ -782,7 +783,6 @@ 'src/core/lib/security/transport/security_handshaker.c', 'src/core/lib/security/transport/server_auth_filter.c', 'src/core/lib/security/transport/tsi_error.c', - 'src/core/lib/security/util/b64.c', 'src/core/lib/security/util/json_util.c', 'src/core/lib/surface/init_secure.c', 'src/core/lib/tsi/fake_transport_security.c', diff --git a/build.yaml b/build.yaml index b11a3cf3190..6e590f4ca6b 100644 --- a/build.yaml +++ b/build.yaml @@ -369,6 +369,7 @@ filegroups: - src/core/lib/json/json_reader.c - src/core/lib/json/json_string.c - src/core/lib/json/json_writer.c + - src/core/lib/security/util/b64.c - src/core/lib/slice/percent_encoding.c - src/core/lib/slice/slice.c - src/core/lib/slice/slice_buffer.c @@ -4135,7 +4136,6 @@ targets: - test/cpp/interop/interop_client.cc - test/cpp/interop/stress_interop_client.cc - test/cpp/interop/stress_test.cc - - test/cpp/util/create_test_channel.cc - test/cpp/util/metrics_server.cc deps: - grpc++_test_util diff --git a/config.m4 b/config.m4 index 94e3ba1ab59..b6f0bff7ef0 100644 --- a/config.m4 +++ b/config.m4 @@ -168,6 +168,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/json/json_reader.c \ src/core/lib/json/json_string.c \ src/core/lib/json/json_writer.c \ + src/core/lib/security/util/b64.c \ src/core/lib/slice/percent_encoding.c \ src/core/lib/slice/slice.c \ src/core/lib/slice/slice_buffer.c \ @@ -250,7 +251,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/security/transport/security_handshaker.c \ src/core/lib/security/transport/server_auth_filter.c \ src/core/lib/security/transport/tsi_error.c \ - src/core/lib/security/util/b64.c \ src/core/lib/security/util/json_util.c \ src/core/lib/surface/init_secure.c \ src/core/lib/tsi/fake_transport_security.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index f9408ab1dbc..8ea9e5fbdec 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -543,6 +543,7 @@ Pod::Spec.new do |s| 'src/core/lib/json/json_reader.c', 'src/core/lib/json/json_string.c', 'src/core/lib/json/json_writer.c', + 'src/core/lib/security/util/b64.c', 'src/core/lib/slice/percent_encoding.c', 'src/core/lib/slice/slice.c', 'src/core/lib/slice/slice_buffer.c', @@ -625,7 +626,6 @@ Pod::Spec.new do |s| 'src/core/lib/security/transport/security_handshaker.c', 'src/core/lib/security/transport/server_auth_filter.c', 'src/core/lib/security/transport/tsi_error.c', - 'src/core/lib/security/util/b64.c', 'src/core/lib/security/util/json_util.c', 'src/core/lib/surface/init_secure.c', 'src/core/lib/tsi/fake_transport_security.c', diff --git a/grpc.gemspec b/grpc.gemspec index 46097cb4e70..06da332bed5 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -459,6 +459,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/json/json_reader.c ) s.files += %w( src/core/lib/json/json_string.c ) s.files += %w( src/core/lib/json/json_writer.c ) + s.files += %w( src/core/lib/security/util/b64.c ) s.files += %w( src/core/lib/slice/percent_encoding.c ) s.files += %w( src/core/lib/slice/slice.c ) s.files += %w( src/core/lib/slice/slice_buffer.c ) @@ -541,7 +542,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/transport/security_handshaker.c ) s.files += %w( src/core/lib/security/transport/server_auth_filter.c ) s.files += %w( src/core/lib/security/transport/tsi_error.c ) - s.files += %w( src/core/lib/security/util/b64.c ) s.files += %w( src/core/lib/security/util/json_util.c ) s.files += %w( src/core/lib/surface/init_secure.c ) s.files += %w( src/core/lib/tsi/fake_transport_security.c ) diff --git a/package.xml b/package.xml index f9407bb3edd..5d585845556 100644 --- a/package.xml +++ b/package.xml @@ -468,6 +468,7 @@ + @@ -550,7 +551,6 @@ - diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index 028e9a5d04e..967904df1ee 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -36,6 +36,7 @@ #include #include #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/security/util/b64.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -56,7 +57,6 @@ typedef struct call_data { grpc_linked_mdelem te_trailers; grpc_linked_mdelem content_type; grpc_linked_mdelem user_agent; - grpc_linked_mdelem payload_bin; grpc_metadata_batch *recv_initial_metadata; grpc_metadata_batch *recv_trailing_metadata; @@ -292,19 +292,58 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, continue_send_message(exec_ctx, elem); if (calld->send_message_blocked == false) { - /* when all the send_message data is available, then create a MDELEM and - append to headers */ - grpc_mdelem payload_bin = grpc_mdelem_from_slices( - exec_ctx, GRPC_MDSTR_GRPC_PAYLOAD_BIN, - grpc_slice_from_copied_buffer((const char *)calld->payload_bytes, - op->send_message->length)); - error = - grpc_metadata_batch_add_tail(exec_ctx, op->send_initial_metadata, - &calld->payload_bin, payload_bin); + /* when all the send_message data is available, then modify the path + * MDELEM by appending base64 encoded query to the path */ + const int k_url_safe = 1; + const int k_multi_line = 0; + const unsigned char k_query_separator = '?'; + + grpc_slice path_slice = + GRPC_MDVALUE(op->send_initial_metadata->idx.named.path->md); + /* sum up individual component's lengths and allocate enough memory to + * hold combined path+query */ + size_t estimated_len = GRPC_SLICE_LENGTH(path_slice); + estimated_len++; /* for the '?' */ + estimated_len += grpc_base64_estimate_encoded_size( + op->send_message->length, k_url_safe, k_multi_line); + estimated_len += 1; /* for the trailing 0 */ + grpc_slice path_with_query_slice = grpc_slice_malloc(estimated_len); + + /* memcopy individual pieces into this slice */ + uint8_t *write_ptr = + (uint8_t *)GRPC_SLICE_START_PTR(path_with_query_slice); + uint8_t *original_path = (uint8_t *)GRPC_SLICE_START_PTR(path_slice); + memcpy(write_ptr, original_path, GRPC_SLICE_LENGTH(path_slice)); + write_ptr += GRPC_SLICE_LENGTH(path_slice); + + *write_ptr = k_query_separator; + write_ptr++; /* for the '?' */ + + grpc_base64_encode_core((char *)write_ptr, calld->payload_bytes, + op->send_message->length, k_url_safe, + k_multi_line); + + /* remove trailing unused memory and add trailing 0 to terminate string + */ + char *t = (char *)GRPC_SLICE_START_PTR(path_with_query_slice); + /* safe to use strlen since base64_encode will always add '\0' */ + size_t path_length = strlen(t) + 1; + *(t + path_length) = '\0'; + path_with_query_slice = + grpc_slice_sub(path_with_query_slice, 0, path_length); + + /* substitute previous path with the new path+query */ + grpc_mdelem mdelem_path_and_query = grpc_mdelem_from_slices( + exec_ctx, GRPC_MDSTR_PATH, path_with_query_slice); + grpc_metadata_batch *b = op->send_initial_metadata; + error = grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, + mdelem_path_and_query); if (error != GRPC_ERROR_NONE) return error; + calld->on_complete = op->on_complete; op->on_complete = &calld->hc_on_complete; op->send_message = NULL; + grpc_slice_unref_internal(exec_ctx, path_with_query_slice); } else { /* Not all data is available. Fall back to POST. */ gpr_log(GPR_DEBUG, diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index 41a325bc04b..8d3c488ea0a 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -37,6 +37,7 @@ #include #include #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/security/util/b64.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -51,8 +52,8 @@ typedef struct call_data { grpc_linked_mdelem status; grpc_linked_mdelem content_type; - /* did this request come with payload-bin */ - bool seen_payload_bin; + /* did this request come with path query containing request payload */ + bool seen_path_with_query; /* flag to ensure payload_bin is delivered only once */ bool payload_bin_delivered; @@ -61,7 +62,7 @@ typedef struct call_data { bool *recv_cacheable_request; /** Closure to call when finished with the hs_on_recv hook */ grpc_closure *on_done_recv; - /** Closure to call when we retrieve read message from the payload-bin header + /** Closure to call when we retrieve read message from the path URI */ grpc_closure *recv_message_ready; grpc_closure *on_complete; @@ -205,6 +206,43 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, grpc_error_set_str( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing header"), GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":path"))); + } else if (*calld->recv_cacheable_request == true) { + /* We have a cacheable request made with GET verb. The path contains the + * query parameter which is base64 encoded request payload. */ + const char k_query_separator = '?'; + grpc_slice path_slice = GRPC_MDVALUE(b->idx.named.path->md); + uint8_t *path_ptr = (uint8_t *)GRPC_SLICE_START_PTR(path_slice); + size_t path_length = GRPC_SLICE_LENGTH(path_slice); + /* offset of the character '?' */ + size_t offset = 0; + for (offset = 0; *path_ptr != k_query_separator && offset < path_length; + path_ptr++, offset++) + ; + if (offset < path_length) { + grpc_slice query_slice = + grpc_slice_sub(path_slice, offset + 1, path_length); + + /* substitute path metadata with just the path (not query) */ + grpc_mdelem mdelem_path_without_query = grpc_mdelem_from_slices( + exec_ctx, GRPC_MDSTR_PATH, grpc_slice_sub(path_slice, 0, offset)); + + grpc_metadata_batch_substitute(exec_ctx, b, b->idx.named.path, + mdelem_path_without_query); + + /* decode payload from query and add to the slice buffer to be returned */ + const int k_url_safe = 1; + grpc_slice_buffer_add( + &calld->read_slice_buffer, + grpc_base64_decode(exec_ctx, + (const char *)GRPC_SLICE_START_PTR(query_slice), + k_url_safe)); + grpc_slice_buffer_stream_init(&calld->read_stream, + &calld->read_slice_buffer, 0); + calld->seen_path_with_query = true; + grpc_slice_unref_internal(exec_ctx, query_slice); + } else { + gpr_log(GPR_ERROR, "GET request without QUERY"); + } } if (b->idx.named.host != NULL && b->idx.named.authority == NULL) { @@ -228,16 +266,6 @@ static grpc_error *server_filter_incoming_metadata(grpc_exec_ctx *exec_ctx, GRPC_ERROR_STR_KEY, grpc_slice_from_static_string(":authority"))); } - if (b->idx.named.grpc_payload_bin != NULL) { - calld->seen_payload_bin = true; - grpc_slice_buffer_add(&calld->read_slice_buffer, - grpc_slice_ref_internal( - GRPC_MDVALUE(b->idx.named.grpc_payload_bin->md))); - grpc_slice_buffer_stream_init(&calld->read_stream, - &calld->read_slice_buffer, 0); - grpc_metadata_batch_remove(exec_ctx, b, b->idx.named.grpc_payload_bin); - } - return error; } @@ -258,8 +286,8 @@ static void hs_on_complete(grpc_exec_ctx *exec_ctx, void *user_data, grpc_error *err) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - /* Call recv_message_ready if we got the payload via the header field */ - if (calld->seen_payload_bin && calld->recv_message_ready != NULL) { + /* Call recv_message_ready if we got the payload via the path field */ + if (calld->seen_path_with_query && calld->recv_message_ready != NULL) { *calld->pp_recv_message = calld->payload_bin_delivered ? NULL : (grpc_byte_stream *)&calld->read_stream; @@ -274,7 +302,7 @@ static void hs_recv_message_ready(grpc_exec_ctx *exec_ctx, void *user_data, grpc_error *err) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - if (calld->seen_payload_bin) { + if (calld->seen_path_with_query) { /* do nothing. This is probably a GET request, and payload will be returned in hs_on_complete callback. */ } else { diff --git a/src/core/lib/security/util/b64.c b/src/core/lib/security/util/b64.c index 09c82131316..0d5a9176602 100644 --- a/src/core/lib/security/util/b64.c +++ b/src/core/lib/security/util/b64.c @@ -71,15 +71,31 @@ static const char base64_url_safe_chars[] = char *grpc_base64_encode(const void *vdata, size_t data_size, int url_safe, int multiline) { - const unsigned char *data = vdata; - const char *base64_chars = - url_safe ? base64_url_safe_chars : base64_url_unsafe_chars; + size_t result_projected_size = + grpc_base64_estimate_encoded_size(data_size, url_safe, multiline); + char *result = gpr_malloc(result_projected_size); + grpc_base64_encode_core(result, vdata, data_size, url_safe, multiline); + return result; +} + +size_t grpc_base64_estimate_encoded_size(size_t data_size, int url_safe, + int multiline) { size_t result_projected_size = 4 * ((data_size + 3) / 3) + 2 * (multiline ? (data_size / (3 * GRPC_BASE64_MULTILINE_NUM_BLOCKS)) : 0) + 1; - char *result = gpr_malloc(result_projected_size); + return result_projected_size; +} + +void grpc_base64_encode_core(char *result, const void *vdata, size_t data_size, + int url_safe, int multiline) { + const unsigned char *data = vdata; + const char *base64_chars = + url_safe ? base64_url_safe_chars : base64_url_unsafe_chars; + const size_t result_projected_size = + grpc_base64_estimate_encoded_size(data_size, url_safe, multiline); + char *current = result; size_t num_blocks = 0; size_t i = 0; @@ -119,7 +135,6 @@ char *grpc_base64_encode(const void *vdata, size_t data_size, int url_safe, GPR_ASSERT(current >= result); GPR_ASSERT((uintptr_t)(current - result) < result_projected_size); result[current - result] = '\0'; - return result; } grpc_slice grpc_base64_decode(grpc_exec_ctx *exec_ctx, const char *b64, diff --git a/src/core/lib/security/util/b64.h b/src/core/lib/security/util/b64.h index d42a136f61a..ef52291c6a4 100644 --- a/src/core/lib/security/util/b64.h +++ b/src/core/lib/security/util/b64.h @@ -37,10 +37,22 @@ #include /* Encodes data using base64. It is the caller's responsability to free - the returned char * using gpr_free. Returns NULL on NULL input. */ + the returned char * using gpr_free. Returns NULL on NULL input. + TODO(makdharma) : change the flags to bool from int */ char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, int multiline); +/* estimate the upper bound on size of base64 encoded data. The actual size + * is guaranteed to be less than or equal to the size returned here. */ +size_t grpc_base64_estimate_encoded_size(size_t data_size, int url_safe, + int multiline); + +/* Encodes data using base64 and write it to memory pointed to by result. It is + * the caller's responsiblity to allocate enough memory in |result| to fit the + * encoded data. */ +void grpc_base64_encode_core(char *result, const void *vdata, size_t data_size, + int url_safe, int multiline); + /* Decodes data according to the base64 specification. Returns an empty slice in case of failure. */ grpc_slice grpc_base64_decode(grpc_exec_ctx *exec_ctx, const char *b64, diff --git a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m index 6fe4706af67..6d94116c759 100644 --- a/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m +++ b/src/objective-c/tests/CronetUnitTests/CronetUnitTests.m @@ -187,6 +187,18 @@ unsigned int parse_h2_length(const char *field) { grpc_metadata_array_init(&request_metadata_recv); grpc_call_details_init(&call_details); + int sl = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sl >= 0); + + // Make an TCP endpoint to accept the connection + struct sockaddr_in s_addr; + memset(&s_addr, 0, sizeof(s_addr)); + s_addr.sin_family = AF_INET; + s_addr.sin_addr.s_addr = htonl(INADDR_ANY); + s_addr.sin_port = htons(port); + GPR_ASSERT(0 == bind(sl, (struct sockaddr *)&s_addr, sizeof(s_addr))); + GPR_ASSERT(0 == listen(sl, 5)); + memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; @@ -226,17 +238,6 @@ unsigned int parse_h2_length(const char *field) { dispatch_async( dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - int sl = socket(AF_INET, SOCK_STREAM, 0); - GPR_ASSERT(sl >= 0); - - // Make and TCP endpoint to accept the connection - struct sockaddr_in s_addr; - memset(&s_addr, 0, sizeof(s_addr)); - s_addr.sin_family = AF_INET; - s_addr.sin_addr.s_addr = htonl(INADDR_ANY); - s_addr.sin_port = htons(port); - GPR_ASSERT(0 == bind(sl, (struct sockaddr *)&s_addr, sizeof(s_addr))); - GPR_ASSERT(0 == listen(sl, 5)); int s = accept(sl, NULL, NULL); GPR_ASSERT(s >= 0); @@ -324,17 +325,18 @@ unsigned int parse_h2_length(const char *field) { __weak XCTestExpectation *expectation = [self expectationWithDescription:@"Coalescing"]; + int sl = socket(AF_INET, SOCK_STREAM, 0); + GPR_ASSERT(sl >= 0); + struct sockaddr_in s_addr; + memset(&s_addr, 0, sizeof(s_addr)); + s_addr.sin_family = AF_INET; + s_addr.sin_addr.s_addr = htonl(INADDR_ANY); + s_addr.sin_port = htons(port); + GPR_ASSERT(0 == bind(sl, (struct sockaddr *)&s_addr, sizeof(s_addr))); + GPR_ASSERT(0 == listen(sl, 5)); + dispatch_async( dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - int sl = socket(AF_INET, SOCK_STREAM, 0); - GPR_ASSERT(sl >= 0); - struct sockaddr_in s_addr; - memset(&s_addr, 0, sizeof(s_addr)); - s_addr.sin_family = AF_INET; - s_addr.sin_addr.s_addr = htonl(INADDR_ANY); - s_addr.sin_port = htons(port); - GPR_ASSERT(0 == bind(sl, (struct sockaddr *)&s_addr, sizeof(s_addr))); - GPR_ASSERT(0 == listen(sl, 5)); int s = accept(sl, NULL, NULL); GPR_ASSERT(s >= 0); struct timeval tv; @@ -389,9 +391,6 @@ unsigned int parse_h2_length(const char *field) { [expectation fulfill]; }); - // Guarantees that server is listening to the port before client connects. - sleep(1); - memset(ops, 0, sizeof(ops)); op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; diff --git a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme index 49dc3faa3d0..d1d616c4cf2 100644 --- a/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme +++ b/src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/AllTests.xcscheme @@ -49,26 +49,6 @@ - - - - - - - - + +#include +#include +#include + +#include "gflags/gflags.h" +#include "mongoose.h" + +// TODO (makdharma): remove local copies of these protos +#include "tools/grpcz/census.grpc.pb.h" +#include "tools/grpcz/monitoring.grpc.pb.h" + +DEFINE_string( + grpcz_server, "127.0.0.1:8080", + "Unix domain socket path (e.g. unix://tmp/grpcz.sock) or IP address" + "(host:port) where grpcz server is running."); +DEFINE_string(http_port, "8000", + "Port id for accessing the HTTP server that renders /grpcz page"); +DEFINE_bool(print_to_console, false, + "print the JSON retreived from grpcz server and quit"); + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; + +using ::grpc::instrumentation::v1alpha::CanonicalRpcStats; +using ::grpc::instrumentation::v1alpha::Monitoring; + +static const std::string static_html_header = + " \ +\ +

GRPCZ Statistics

\ + "; + +class GrpczClient { + public: + GrpczClient(std::shared_ptr channel) + : stub_(Monitoring::NewStub(channel)) {} + + std::string GetStatsAsJson() { + const ::google::protobuf::Empty request; + CanonicalRpcStats reply; + ClientContext context; + Status status = stub_->GetCanonicalRpcStats(&context, request, &reply); + + if (status.ok()) { + std::string json_str; + ::google::protobuf::util::MessageToJsonString(reply, &json_str); + return json_str; + } else { + static const std::string error_message_json = + "{\"Error Message\":\"" + status.error_message() + "\"}"; + gpr_log(GPR_DEBUG, "%d: %s", status.error_code(), + status.error_message().c_str()); + return error_message_json; + } + } + + private: + std::unique_ptr stub_; +}; + +static struct mg_serve_http_opts s_http_server_opts; +std::unique_ptr g_grpcz_client; + +static void ev_handler(struct mg_connection *nc, int ev, void *p) { + if (ev == MG_EV_HTTP_REQUEST) { + mg_serve_http(nc, (struct http_message *)p, s_http_server_opts); + } +} + +static void grpcz_handler(struct mg_connection *nc, int ev, void *ev_data) { + (void)ev; + (void)ev_data; + gpr_log(GPR_INFO, "fetching grpcz stats from %s", FLAGS_grpcz_server.c_str()); + std::string json_str = g_grpcz_client->GetStatsAsJson(); + std::string rendered_html = + static_html_header + json_str + static_html_footer; + mg_printf(nc, "HTTP/1.0 200 OK\r\n\r\n%s", rendered_html.c_str()); + nc->flags |= MG_F_SEND_AND_CLOSE; +} + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + + // Create a client + g_grpcz_client.reset(new GrpczClient(grpc::CreateChannel( + FLAGS_grpcz_server, grpc::InsecureChannelCredentials()))); + if (FLAGS_print_to_console) { + // using GPR_ERROR since this is the default verbosity. _DEBUG or _INFO + // won't print unless GRPC_VERBOSITY env var is set appropriately, which + // might confuse users of this utility. + gpr_log(GPR_ERROR, "%s\n", g_grpcz_client->GetStatsAsJson().c_str()); + return 0; + } + + // Set up a mongoose webserver handler + struct mg_mgr mgr; + mg_mgr_init(&mgr, NULL); + gpr_log(GPR_INFO, "Starting grpcz web server on port %s\n", + FLAGS_http_port.c_str()); + + struct mg_connection *nc = mg_bind(&mgr, FLAGS_http_port.c_str(), ev_handler); + if (nc == NULL) { + gpr_log(GPR_ERROR, "Failed to create listener on port %s\n", + FLAGS_http_port.c_str()); + return -1; + } + mg_register_http_endpoint(nc, "/grpcz", grpcz_handler); + mg_set_protocol_http_websocket(nc); + + // Poll in a loop and serve /grpcz pages + for (;;) { + static const int k_sleep_millis = 100; + mg_mgr_poll(&mgr, k_sleep_millis); + } + mg_mgr_free(&mgr); + return 0; +} diff --git a/tools/grpcz/monitoring.proto b/tools/grpcz/monitoring.proto new file mode 100644 index 00000000000..fefcd7d22f1 --- /dev/null +++ b/tools/grpcz/monitoring.proto @@ -0,0 +1,156 @@ +// Copyright 2017, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. +syntax = "proto3"; + +// TODO(ericgribkoff) Figure out how to manage the external Census proto +// dependency. +import "tools/grpcz/census.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/empty.proto"; + +package grpc.instrumentation.v1alpha; + +option java_multiple_files = true; +option java_package = "io.grpc.instrumentation.v1alpha"; +option java_outer_classname = "MonitoringProto"; + +service Monitoring { + // Return canonical RPC stats + rpc GetCanonicalRpcStats(google.protobuf.Empty) returns (CanonicalRpcStats) { + } + + // Query the server for specific stats + rpc GetStats(StatsRequest) returns (StatsResponse) { + } + + // Request the server to stream back snapshots of the requested stats + rpc WatchStats(StatsRequest) returns (stream StatsResponse) { + } + + + // Return request traces. + rpc GetRequestTraces(TraceRequest) returns(TraceResponse) { + // TODO(aveitch): Please define the messages here + } + + // Return application-defined groups of monitoring data. + // This is a low level facility to allow extension of the monitoring API to + // application-specific monitoring data. Frameworks may use this to define + // additional groups of monitoring data made available by servers. + rpc GetCustomMonitoringData(MonitoringDataGroup) + returns (CustomMonitoringData) { + } + +} + +// Canonical RPC stats exported by gRPC. +message CanonicalRpcStats { + StatsResponse rpc_client_errors = 1; + StatsResponse rpc_client_completed_rpcs = 2; + StatsResponse rpc_client_started_rpcs = 3; + StatsResponse rpc_client_elapsed_time = 4; + StatsResponse rpc_client_server_elapsed_time = 5; + StatsResponse rpc_client_request_bytes = 6; + StatsResponse rpc_client_response_bytes = 7; + StatsResponse rpc_client_request_count = 8; + StatsResponse rpc_client_response_count = 9; + StatsResponse rpc_server_errors = 10; + StatsResponse rpc_server_completed_rpcs = 11; + StatsResponse rpc_server_server_elapsed_time = 12; + StatsResponse rpc_server_request_bytes = 13; + StatsResponse rpc_server_response_bytes = 14; + StatsResponse rpc_server_request_count = 15; + StatsResponse rpc_server_response_count = 16; + StatsResponse rpc_server_elapsed_time = 17; + //TODO(ericgribkoff) Add minute-hour interval stats. +} + +// This message is sent when requesting a set of stats (Census Views) from +// a client system, as part of the MonitoringService API's. +message StatsRequest { + // An optional set of ViewDescriptor names. Only Views using these + // descriptors will be sent back in the response. If no names are provided, + // then all Views present in the client system will be included in every + // response. If measurement_names is also provided, then Views matching the + // intersection of the two are returned. + // TODO(aveitch): Consider making this a list of regexes or prefix matches in + // the future. + repeated string view_names = 1; + + // An optional set of MeasurementDescriptor names. Only Views using these + // descriptors will be sent back in the response. If no names are provided, + // then all Views present in the client system will be included in every + // response. If view_names is also provided, then Views matching the + // intersection of the two are returned. + // TODO(aveitch): Consider making this a list of regexes or prefix matches in + // the future. + repeated string measurement_names = 2; + + // By default, the MeasurementDescriptors and ViewDescriptors corresponding to + // the Views that are returned in a StatsResponse will be included in the + // first such response. Set this to true to have them not sent. + bool dont_include_descriptors_in_first_response = 3; +} + +// This message contains all information relevant to a single View. It is the +// return type for GetStats and WatchStats, and used in CanonicalRpcStats. +message StatsResponse { + // A StatsResponse can optionally contain the MeasurementDescriptor and + // ViewDescriptor for the View. These will be sent in the first WatchStats + // response, or all GetStats and GetCanonicalRpcStats responses. These will + // not be set for {Get,Watch}Stats if + // dont_include_descriptors_in_first_response is set to true in the + // StatsRequest. + google.instrumentation.MeasurementDescriptor measurement_descriptor = 1; + google.instrumentation.ViewDescriptor view_descriptor = 2; + + // The View data. + google.instrumentation.View view = 3; +} + +message TraceRequest { + // TODO(aveitch): Complete definition of this type +} + +message TraceResponse { + // TODO(aveitch): Complete definition of this type +} + +message MonitoringDataGroup { + string name = 1; // name of a group of monitoring data +} + +// The wrapper for custom monitoring data. +message CustomMonitoringData { + // can be any application specific monitoring data + google.protobuf.Any contents = 1; +} diff --git a/tools/jenkins/comment_on_pr.sh b/tools/jenkins/comment_on_pr.sh new file mode 100755 index 00000000000..85f33aa917a --- /dev/null +++ b/tools/jenkins/comment_on_pr.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins to comment $1 on pull requests +# when triggered by a build + +set -e + +if [ -z "$1" ] || [ -z $JENKINS_OAUTH_TOKEN ] || [ -z $ghprbPullId ]; then + echo "Insufficient arguments or environment variables provided." + exit 1 +fi + +# Format the comment message to JSON +COMMENT_MESSAGE="{\"body\":\"$1\"}" + +curl -k -H "Authorization: token $JENKINS_OAUTH_TOKEN" -H "Content-Type: application/json" \ + -d "$COMMENT_MESSAGE" https://api.github.com/repos/grpc/grpc/issues/$ghprbPullId/comments diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 99b920f6a09..2b05a8d04d0 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -31,7 +31,12 @@ # This script is invoked by Jenkins and runs performance smoke test. set -ex +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + # Enter the gRPC repo root cd $(dirname $0)/../.. tools/run_tests/run_performance_tests.py -l c++ node ruby csharp python --netperf --category smoketest +# todo(mattkwong): Change performance test to use microbenchmarking +# tools/run_tests/run_microbenchmark.py -c summary --diff_perf origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index e3190152447..fd12ea5c471 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3810,7 +3810,6 @@ "test/cpp/interop/stress_interop_client.cc", "test/cpp/interop/stress_interop_client.h", "test/cpp/interop/stress_test.cc", - "test/cpp/util/create_test_channel.cc", "test/cpp/util/create_test_channel.h", "test/cpp/util/metrics_server.cc", "test/cpp/util/metrics_server.h" @@ -7787,6 +7786,7 @@ "src/core/lib/json/json_string.c", "src/core/lib/json/json_writer.c", "src/core/lib/json/json_writer.h", + "src/core/lib/security/util/b64.c", "src/core/lib/slice/percent_encoding.c", "src/core/lib/slice/percent_encoding.h", "src/core/lib/slice/slice.c", diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 6bedc54941a..19cf16ed499 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -38,6 +38,19 @@ import argparse import python_utils.jobset as jobset import python_utils.start_port_server as start_port_server +_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', + 'bm_closure', + 'bm_cq', + 'bm_call_create', + 'bm_error', + 'bm_chttp2_hpack', + 'bm_chttp2_transport', + 'bm_pollset', + 'bm_metadata', + 'bm_fullstack_trickle'] + flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) @@ -201,19 +214,8 @@ argp.add_argument('-c', '--collect', default=sorted(collectors.keys()), help='Which collectors should be run against each benchmark') argp.add_argument('-b', '--benchmarks', - default=['bm_fullstack_unary_ping_pong', - 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', - 'bm_closure', - 'bm_cq', - 'bm_call_create', - 'bm_error', - 'bm_chttp2_hpack', - 'bm_chttp2_transport', - 'bm_pollset', - 'bm_metadata', - 'bm_fullstack_trickle', - ], + choices=_AVAILABLE_BENCHMARK_TESTS, + default=_AVAILABLE_BENCHMARK_TESTS, nargs='+', type=str, help='Which microbenchmarks should be run') @@ -237,6 +239,7 @@ try: for bm_name in args.benchmarks: collectors[collect](bm_name, args) if args.diff_perf: + git_comment = 'Performance differences between this PR and %s\\n' % args.diff_perf if 'summary' not in args.collect: for bm_name in args.benchmarks: run_summary(bm_name, 'opt', bm_name) @@ -264,7 +267,14 @@ try: if diff: heading('Performance diff: %s' % bm_name) text(diff) + git_comment += '```\\nPerformance diff: %s\\n%s\\n```\\n' % (bm_name, diff.replace('\n', '\\n')) finally: + if args.diff_perf: + subprocess.call(['tools/jenkins/comment_on_pr.sh "%s"' % git_comment.replace('`', '\`')], + stdout=subprocess.PIPE, + shell=True) + if not os.path.exists('reports'): + os.makedirs('reports') index_html += "\n\n" with open('reports/index.html', 'w') as f: f.write(index_html) diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 0b2127f308c..7abff218dde 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -675,6 +675,8 @@ + + @@ -839,8 +841,6 @@ - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 0a3c9c621d0..aa3e5a77aa6 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -259,6 +259,9 @@ src\core\lib\json + + src\core\lib\security\util + src\core\lib\slice @@ -505,9 +508,6 @@ src\core\lib\security\transport - - src\core\lib\security\util - src\core\lib\security\util diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 04d6d9f55a3..9f3743b238e 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -516,6 +516,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index a2849888a14..bc52644206e 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -316,6 +316,9 @@ src\core\lib\json + + src\core\lib\security\util + src\core\lib\slice @@ -941,6 +944,12 @@ {89bc8f83-e29a-ddab-8f6b-22df11cdc867} + + {83d9c8a0-c579-769e-e2df-270c67b8c159} + + + {d55417ff-35a3-ef69-80c4-e6a95673728b} + {4d172bbc-20c4-6e7d-872a-2d287b589aa0} diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 86549398dc8..1a0c38cecbf 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -642,6 +642,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 625f276a55e..b11766ecfd9 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -262,6 +262,9 @@ src\core\lib\json + + src\core\lib\security\util + src\core\lib\slice @@ -1352,6 +1355,12 @@ {681cdaeb-c47f-8853-d985-bf13c2873947} + + {02c849c5-66eb-e42f-c9f7-2cf7cc9db06a} + + + {c22aef1f-8b7b-6b5c-9e22-321b11f6352d} + {74c81ab7-e329-a362-3890-4c41b90f0511} diff --git a/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj b/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj index fed916f50a9..2944a31c483 100644 --- a/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj +++ b/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj @@ -205,8 +205,6 @@ - - diff --git a/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj.filters b/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj.filters index 9339621c8d1..83dc09a1d4b 100644 --- a/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj.filters +++ b/vsprojects/vcxproj/test/stress_test/stress_test.vcxproj.filters @@ -22,9 +22,6 @@ test\cpp\interop - - test\cpp\util - test\cpp\util