From 3a48847182cf6c79499a4e9c9a2c6ec526f5e185 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Feb 2017 14:35:26 -0800 Subject: [PATCH 001/105] qps BUILD progress --- test/cpp/qps/BUILD | 112 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 test/cpp/qps/BUILD diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD new file mode 100644 index 00000000000..6f3265bdd43 --- /dev/null +++ b/test/cpp/qps/BUILD @@ -0,0 +1,112 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +licenses(["notice"]) # 3-clause BSD + +cc_library( + name = "parse_json", + hdrs = ["parse_json.h"], + srcs = ["parse_json.cc"], + deps = ["//:grpc++"], +) + +cc_library( + name = "qps_worker_impl", + hdrs = ["client.h", "server.h", "qps_worker.h"], + srcs = ["client_async.cc", "client_sync.cc", "server_async.cc", "server_sync.cc", "qps_worker.cc"], + deps = ["//:grpc++", + "//:grpc", + "//test/core/util:grpc_test_util", + "//test/core/util:gpr_test_util", + "//src/proto/grpc/testing:services_proto", + "//src/proto/grpc/testing:payloads_proto", + "//src/proto/grpc/testing:control_proto", + "//external:gtest"], +) + +cc_library( + name = "driver_impl", + hdrs = ["driver.h", "report.h"], + srcs = ["driver.cc", "report.cc"], + deps = ["//:grpc++"] +) + +cc_library( + name = "histogram", + hdrs = ["histogram.h", "stats.h"], + deps = ["//:gpr"] +) + +cc_library( + name = "interarrival", + hdrs = ["interarrival.h"], + deps = ["//:grpc++"] +) + +cc_binary( + name = "json_run_localhost", + srcs = ["json_run_localhost.cc"], + deps = ["//:gpr", "//test/core/util:grpc_test_util", "//test/core/util:gpr_test_util", "//test/cpp/util:test_util"] +) + +cc_test( + name = "qps_interarrival_test", + srcs = ["qps_interarrival_test.cc"], + deps = [":histogram", ":interarrival"] +) + +cc_binary( + name = "qps_json_driver", + srcs = ["qps_json_driver.cc"], +) + +cc_test( + name = "qps_openloop_test", + srcs = ["qps_openloop_test.cc"], + deps = [":qps_worker_impl", ":driver_impl"] +) + +cc_test( + name = "secure_sync_unary_ping_pong_test", + srcs = ["secure_sync_unary_ping_pong_test.cc"], + deps = ["//:grpc++", ":driver_impl"] +) + +cc_library( + name = "usage_timer", + srcs = ["usage_timer.cc"], + hdrs = ["usage_timer.h"], + deps = ["//:gpr"] +) + +cc_binary( + name = "qps_worker", + srcs = ["worker.cc"], + deps = ["//:grpc++", ":qps_worker_impl", "//test/core/util:grpc_test_util", "//test/core/util:gpr_test_util", "//test/cpp/util:test_util", "//test/cpp/util:test_config"] +) From bbc1eb00a86fdd5d3bea0ede8060563f45a4188e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Feb 2017 14:40:40 -0800 Subject: [PATCH 002/105] Finish test/cpp/common --- test/cpp/common/BUILD | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/cpp/common/BUILD b/test/cpp/common/BUILD index 0e2db00f0ab..48ad5839813 100644 --- a/test/cpp/common/BUILD +++ b/test/cpp/common/BUILD @@ -34,3 +34,27 @@ cc_test( srcs = ["alarm_cpp_test.cc"], deps = ["//:grpc++", "//external:gtest", "//test/core/util:gpr_test_util"], ) + +cc_test( + name = "auth_property_iterator_test", + srcs = ["auth_property_iterator_test.cc"], + deps = ["//:grpc++", "//external:gtest", "//test/core/util:gpr_test_util", "//test/cpp/util:test_util"], +) + +cc_test( + name = "channel_arguments_test", + srcs = ["channel_arguments_test.cc"], + deps = ["//:grpc++", "//external:gtest", "//test/core/util:gpr_test_util"], +) + +cc_test( + name = "channel_filter_test", + srcs = ["channel_filter_test.cc"], + deps = ["//:grpc++", "//external:gtest", "//test/core/util:gpr_test_util"], +) + +cc_test( + name = "secure_auth_context_test", + srcs = ["secure_auth_context_test.cc"], + deps = ["//:grpc++", "//external:gtest", "//test/core/util:gpr_test_util", "//test/cpp/util:test_util"], +) From 19f3ea2c204794e7eabc5bbc34fde3328fedb22a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Feb 2017 15:17:05 -0800 Subject: [PATCH 003/105] Start bringing microbenchmarks to Bazel --- Makefile | 99 +++++++++++++++++++++- WORKSPACE | 11 +++ build.yaml | 3 + templates/Makefile.template | 9 +- test/cpp/microbenchmarks/BUILD | 36 ++++++++ test/cpp/microbenchmarks/noop-benchmark.cc | 2 +- 6 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 test/cpp/microbenchmarks/BUILD diff --git a/Makefile b/Makefile index 0acf06e946d..fb251115724 100644 --- a/Makefile +++ b/Makefile @@ -465,7 +465,7 @@ SHARED_VERSION_CORE = -3 SHARED_VERSION_CPP = -1 SHARED_VERSION_CSHARP = -1 else ifeq ($(SYSTEM),Darwin) -EXECUTABLE_SUFFIX = +EXECUTABLE_SUFFIX = SHARED_EXT_CORE = dylib SHARED_EXT_CPP = dylib SHARED_EXT_CSHARP = dylib @@ -474,7 +474,7 @@ SHARED_VERSION_CORE = SHARED_VERSION_CPP = SHARED_VERSION_CSHARP = else -EXECUTABLE_SUFFIX = +EXECUTABLE_SUFFIX = SHARED_EXT_CORE = so.$(CORE_VERSION) SHARED_EXT_CPP = so.$(CPP_VERSION) SHARED_EXT_CSHARP = so.$(CSHARP_VERSION) @@ -12384,6 +12384,7 @@ endif endif +$(BM_CLOSURE_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_closure: $(BM_CLOSURE_OBJS:.o=.dep) @@ -12427,6 +12428,7 @@ endif endif +$(BM_FULLSTACK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep) @@ -13914,6 +13916,7 @@ endif endif +$(NOOP-BENCHMARK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/noop-benchmark.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a deps_noop-benchmark: $(NOOP-BENCHMARK_OBJS:.o=.dep) @@ -14980,6 +14983,8 @@ $(BINDIR)/$(CONFIG)/boringssl_aes_test: $(LIBDIR)/$(CONFIG)/libboringssl_aes_te endif +$(BORINGSSL_AES_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_AES_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15007,6 +15012,8 @@ $(BINDIR)/$(CONFIG)/boringssl_asn1_test: $(LIBDIR)/$(CONFIG)/libboringssl_asn1_ endif +$(BORINGSSL_ASN1_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ASN1_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15034,6 +15041,8 @@ $(BINDIR)/$(CONFIG)/boringssl_base64_test: $(LIBDIR)/$(CONFIG)/libboringssl_bas endif +$(BORINGSSL_BASE64_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_BASE64_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15061,6 +15070,8 @@ $(BINDIR)/$(CONFIG)/boringssl_bio_test: $(LIBDIR)/$(CONFIG)/libboringssl_bio_te endif +$(BORINGSSL_BIO_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_BIO_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15088,6 +15099,8 @@ $(BINDIR)/$(CONFIG)/boringssl_bn_test: $(LIBDIR)/$(CONFIG)/libboringssl_bn_test endif +$(BORINGSSL_BN_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_BN_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15115,6 +15128,8 @@ $(BINDIR)/$(CONFIG)/boringssl_bytestring_test: $(LIBDIR)/$(CONFIG)/libboringssl endif +$(BORINGSSL_BYTESTRING_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_BYTESTRING_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15142,6 +15157,8 @@ $(BINDIR)/$(CONFIG)/boringssl_chacha_test: $(LIBDIR)/$(CONFIG)/libboringssl_cha endif +$(BORINGSSL_CHACHA_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_CHACHA_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15169,6 +15186,8 @@ $(BINDIR)/$(CONFIG)/boringssl_aead_test: $(LIBDIR)/$(CONFIG)/libboringssl_aead_ endif +$(BORINGSSL_AEAD_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_AEAD_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15196,6 +15215,8 @@ $(BINDIR)/$(CONFIG)/boringssl_cipher_test: $(LIBDIR)/$(CONFIG)/libboringssl_cip endif +$(BORINGSSL_CIPHER_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_CIPHER_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15223,6 +15244,8 @@ $(BINDIR)/$(CONFIG)/boringssl_cmac_test: $(LIBDIR)/$(CONFIG)/libboringssl_cmac_ endif +$(BORINGSSL_CMAC_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_CMAC_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15250,6 +15273,8 @@ $(BINDIR)/$(CONFIG)/boringssl_constant_time_test: $(LIBDIR)/$(CONFIG)/libboring endif +$(BORINGSSL_CONSTANT_TIME_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_CONSTANT_TIME_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15277,6 +15302,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ed25519_test: $(LIBDIR)/$(CONFIG)/libboringssl_ed endif +$(BORINGSSL_ED25519_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ED25519_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15304,6 +15331,8 @@ $(BINDIR)/$(CONFIG)/boringssl_spake25519_test: $(LIBDIR)/$(CONFIG)/libboringssl endif +$(BORINGSSL_SPAKE25519_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_SPAKE25519_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15331,6 +15360,8 @@ $(BINDIR)/$(CONFIG)/boringssl_x25519_test: $(LIBDIR)/$(CONFIG)/libboringssl_x25 endif +$(BORINGSSL_X25519_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_X25519_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15358,6 +15389,8 @@ $(BINDIR)/$(CONFIG)/boringssl_dh_test: $(LIBDIR)/$(CONFIG)/libboringssl_dh_test endif +$(BORINGSSL_DH_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_DH_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15385,6 +15418,8 @@ $(BINDIR)/$(CONFIG)/boringssl_digest_test: $(LIBDIR)/$(CONFIG)/libboringssl_dig endif +$(BORINGSSL_DIGEST_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_DIGEST_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15412,6 +15447,8 @@ $(BINDIR)/$(CONFIG)/boringssl_dsa_test: $(LIBDIR)/$(CONFIG)/libboringssl_dsa_te endif +$(BORINGSSL_DSA_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_DSA_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15439,6 +15476,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ec_test: $(LIBDIR)/$(CONFIG)/libboringssl_ec_test endif +$(BORINGSSL_EC_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_EC_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15466,6 +15505,8 @@ $(BINDIR)/$(CONFIG)/boringssl_example_mul: $(LIBDIR)/$(CONFIG)/libboringssl_exa endif +$(BORINGSSL_EXAMPLE_MUL_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_EXAMPLE_MUL_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15493,6 +15534,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ecdh_test: $(LIBDIR)/$(CONFIG)/libboringssl_ecdh_ endif +$(BORINGSSL_ECDH_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ECDH_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15520,6 +15563,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ecdsa_sign_test: $(LIBDIR)/$(CONFIG)/libboringssl endif +$(BORINGSSL_ECDSA_SIGN_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ECDSA_SIGN_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15547,6 +15592,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ecdsa_test: $(LIBDIR)/$(CONFIG)/libboringssl_ecds endif +$(BORINGSSL_ECDSA_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ECDSA_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15574,6 +15621,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ecdsa_verify_test: $(LIBDIR)/$(CONFIG)/libborings endif +$(BORINGSSL_ECDSA_VERIFY_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ECDSA_VERIFY_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15601,6 +15650,8 @@ $(BINDIR)/$(CONFIG)/boringssl_err_test: $(LIBDIR)/$(CONFIG)/libboringssl_err_te endif +$(BORINGSSL_ERR_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_ERR_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15628,6 +15679,8 @@ $(BINDIR)/$(CONFIG)/boringssl_evp_extra_test: $(LIBDIR)/$(CONFIG)/libboringssl_ endif +$(BORINGSSL_EVP_EXTRA_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_EVP_EXTRA_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15655,6 +15708,8 @@ $(BINDIR)/$(CONFIG)/boringssl_evp_test: $(LIBDIR)/$(CONFIG)/libboringssl_evp_te endif +$(BORINGSSL_EVP_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_EVP_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15682,6 +15737,8 @@ $(BINDIR)/$(CONFIG)/boringssl_pbkdf_test: $(LIBDIR)/$(CONFIG)/libboringssl_pbkd endif +$(BORINGSSL_PBKDF_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_PBKDF_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15709,6 +15766,8 @@ $(BINDIR)/$(CONFIG)/boringssl_hkdf_test: $(LIBDIR)/$(CONFIG)/libboringssl_hkdf_ endif +$(BORINGSSL_HKDF_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_HKDF_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15736,6 +15795,8 @@ $(BINDIR)/$(CONFIG)/boringssl_hmac_test: $(LIBDIR)/$(CONFIG)/libboringssl_hmac_ endif +$(BORINGSSL_HMAC_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_HMAC_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15763,6 +15824,8 @@ $(BINDIR)/$(CONFIG)/boringssl_lhash_test: $(LIBDIR)/$(CONFIG)/libboringssl_lhas endif +$(BORINGSSL_LHASH_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_LHASH_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15790,6 +15853,8 @@ $(BINDIR)/$(CONFIG)/boringssl_gcm_test: $(LIBDIR)/$(CONFIG)/libboringssl_gcm_te endif +$(BORINGSSL_GCM_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_GCM_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15817,6 +15882,8 @@ $(BINDIR)/$(CONFIG)/boringssl_newhope_statistical_test: $(LIBDIR)/$(CONFIG)/lib endif +$(BORINGSSL_NEWHOPE_STATISTICAL_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_NEWHOPE_STATISTICAL_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15844,6 +15911,8 @@ $(BINDIR)/$(CONFIG)/boringssl_newhope_test: $(LIBDIR)/$(CONFIG)/libboringssl_ne endif +$(BORINGSSL_NEWHOPE_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_NEWHOPE_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15871,6 +15940,8 @@ $(BINDIR)/$(CONFIG)/boringssl_newhope_vectors_test: $(LIBDIR)/$(CONFIG)/libbori endif +$(BORINGSSL_NEWHOPE_VECTORS_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_NEWHOPE_VECTORS_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15898,6 +15969,8 @@ $(BINDIR)/$(CONFIG)/boringssl_obj_test: $(LIBDIR)/$(CONFIG)/libboringssl_obj_te endif +$(BORINGSSL_OBJ_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_OBJ_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15925,6 +15998,8 @@ $(BINDIR)/$(CONFIG)/boringssl_pkcs12_test: $(LIBDIR)/$(CONFIG)/libboringssl_pkc endif +$(BORINGSSL_PKCS12_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_PKCS12_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15952,6 +16027,8 @@ $(BINDIR)/$(CONFIG)/boringssl_pkcs8_test: $(LIBDIR)/$(CONFIG)/libboringssl_pkcs endif +$(BORINGSSL_PKCS8_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_PKCS8_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -15979,6 +16056,8 @@ $(BINDIR)/$(CONFIG)/boringssl_poly1305_test: $(LIBDIR)/$(CONFIG)/libboringssl_p endif +$(BORINGSSL_POLY1305_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_POLY1305_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16006,6 +16085,8 @@ $(BINDIR)/$(CONFIG)/boringssl_refcount_test: $(LIBDIR)/$(CONFIG)/libboringssl_r endif +$(BORINGSSL_REFCOUNT_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_REFCOUNT_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16033,6 +16114,8 @@ $(BINDIR)/$(CONFIG)/boringssl_rsa_test: $(LIBDIR)/$(CONFIG)/libboringssl_rsa_te endif +$(BORINGSSL_RSA_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_RSA_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16060,6 +16143,8 @@ $(BINDIR)/$(CONFIG)/boringssl_thread_test: $(LIBDIR)/$(CONFIG)/libboringssl_thr endif +$(BORINGSSL_THREAD_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_THREAD_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16087,6 +16172,8 @@ $(BINDIR)/$(CONFIG)/boringssl_pkcs7_test: $(LIBDIR)/$(CONFIG)/libboringssl_pkcs endif +$(BORINGSSL_PKCS7_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_PKCS7_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16114,6 +16201,8 @@ $(BINDIR)/$(CONFIG)/boringssl_x509_test: $(LIBDIR)/$(CONFIG)/libboringssl_x509_ endif +$(BORINGSSL_X509_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_X509_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16141,6 +16230,8 @@ $(BINDIR)/$(CONFIG)/boringssl_tab_test: $(LIBDIR)/$(CONFIG)/libboringssl_tab_te endif +$(BORINGSSL_TAB_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_TAB_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16168,6 +16259,8 @@ $(BINDIR)/$(CONFIG)/boringssl_v3name_test: $(LIBDIR)/$(CONFIG)/libboringssl_v3n endif +$(BORINGSSL_V3NAME_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_V3NAME_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) @@ -16195,6 +16288,8 @@ $(BINDIR)/$(CONFIG)/boringssl_ssl_test: $(LIBDIR)/$(CONFIG)/libboringssl_ssl_te endif +$(BORINGSSL_SSL_TEST_OBJS): CPPFLAGS += -Ithird_party/boringssl/include -fvisibility=hidden -DOPENSSL_NO_ASM -D_GNU_SOURCE -DWIN32_LEAN_AND_MEAN -D_HAS_EXCEPTIONS=0 -DNOMINMAX +$(BORINGSSL_SSL_TEST_OBJS): CFLAGS += -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI) diff --git a/WORKSPACE b/WORKSPACE index 4f90f06d881..b92986127b6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -33,6 +33,11 @@ bind( actual = "@submodule_gtest//:gtest", ) +bind( + name = "benchmark", + actual = "@submodule_benchmark//:benchmark", +) + new_local_repository( name = "submodule_boringssl", path = "third_party/boringssl-with-bazel", @@ -56,3 +61,9 @@ new_local_repository( path = "third_party/googletest", build_file = "third_party/gtest.BUILD", ) + +new_local_repository( + name = "submodule_benchmark", + path = "third_party/benchmark", + build_file = "third_party/benchmark.BUILD", +) diff --git a/build.yaml b/build.yaml index 5782afb62e3..cf5a39591c8 100644 --- a/build.yaml +++ b/build.yaml @@ -2998,6 +2998,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3017,6 +3018,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3485,6 +3487,7 @@ targets: - test/cpp/microbenchmarks/noop-benchmark.cc deps: - benchmark + defaults: benchmark - name: proto_server_reflection_test gtest: true build: test diff --git a/templates/Makefile.template b/templates/Makefile.template index 10d7861f03d..525ede685cd 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -405,7 +405,7 @@ SHARED_VERSION_CPP = -${settings.cpp_version.major} SHARED_VERSION_CSHARP = -${settings.csharp_version.major} else ifeq ($(SYSTEM),Darwin) - EXECUTABLE_SUFFIX = + EXECUTABLE_SUFFIX = SHARED_EXT_CORE = dylib SHARED_EXT_CPP = dylib SHARED_EXT_CSHARP = dylib @@ -414,7 +414,7 @@ SHARED_VERSION_CPP = SHARED_VERSION_CSHARP = else - EXECUTABLE_SUFFIX = + EXECUTABLE_SUFFIX = SHARED_EXT_CORE = so.$(CORE_VERSION) SHARED_EXT_CPP = so.$(CPP_VERSION) SHARED_EXT_CSHARP = so.$(CSHARP_VERSION) @@ -1721,6 +1721,11 @@ endif % endif + % if tgt.get('defaults', None): + % for name, value in defaults.get(tgt.defaults).iteritems(): + $(${tgt.name.upper()}_OBJS): ${name} += ${value} + % endfor + % endif % for src in tgt.src: $(OBJDIR)/$(CONFIG)/${os.path.splitext(src)[0]}.o: \ % for dep in tgt.deps: diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD new file mode 100644 index 00000000000..0efa88c9ab6 --- /dev/null +++ b/test/cpp/microbenchmarks/BUILD @@ -0,0 +1,36 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +licenses(["notice"]) # 3-clause BSD + +cc_test( + name = "noop-benchmark", + srcs = ["noop-benchmark.cc"], + deps = ["//:grpc++", "//external:benchmark"], +) diff --git a/test/cpp/microbenchmarks/noop-benchmark.cc b/test/cpp/microbenchmarks/noop-benchmark.cc index 99fa6d5f6e5..7372ad04f25 100644 --- a/test/cpp/microbenchmarks/noop-benchmark.cc +++ b/test/cpp/microbenchmarks/noop-benchmark.cc @@ -34,7 +34,7 @@ /* This benchmark exists to ensure that the benchmark integration is * working */ -#include "third_party/benchmark/include/benchmark/benchmark.h" +#include static void BM_NoOp(benchmark::State& state) { while (state.KeepRunning()) { From f4ea01b71a041159ecfe16bb5c31159915a0bd8c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Feb 2017 15:21:11 -0800 Subject: [PATCH 004/105] Complete microbenchmarks --- .clang_complete | 1 + test/core/util/BUILD | 2 ++ test/cpp/microbenchmarks/BUILD | 14 +++++++++++++- test/cpp/microbenchmarks/bm_closure.cc | 3 +-- test/cpp/microbenchmarks/bm_fullstack.cc | 2 +- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.clang_complete b/.clang_complete index 79d0946b332..e35f74198f6 100644 --- a/.clang_complete +++ b/.clang_complete @@ -4,5 +4,6 @@ -Igens -I. -Ithird_party/boringssl/include +-Ithird_party/benchmark/include -Ithird_party/zlib -Ithird_party/protobuf/src diff --git a/test/core/util/BUILD b/test/core/util/BUILD index 5c90f1db44b..9c02bc50f81 100644 --- a/test/core/util/BUILD +++ b/test/core/util/BUILD @@ -57,9 +57,11 @@ cc_library( "reconnect_server.c", "slice_splitter.c", "test_tcp_server.c", + "trickle_endpoint.c", ], hdrs = [ "debugger_macros.h", + "trickle_endpoint.h", "grpc_profiler.h", "mock_endpoint.h", "parse_hexstring.h", diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 0efa88c9ab6..b391f8766fc 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -32,5 +32,17 @@ licenses(["notice"]) # 3-clause BSD cc_test( name = "noop-benchmark", srcs = ["noop-benchmark.cc"], - deps = ["//:grpc++", "//external:benchmark"], + deps = ["//external:benchmark"], +) + +cc_test( + name = "bm_fullstack", + srcs = ["bm_fullstack.cc"], + deps = ["//:grpc++", "//external:benchmark", "//test/core/util:grpc_test_util", "//src/proto/grpc/testing:echo_proto"], +) + +cc_test( + name = "bm_closure", + srcs = ["bm_closure.cc"], + deps = ["//:grpc", "//external:benchmark"], ) diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc index 80d6610e13b..586221945aa 100644 --- a/test/cpp/microbenchmarks/bm_closure.cc +++ b/test/cpp/microbenchmarks/bm_closure.cc @@ -33,6 +33,7 @@ /* Test various closure related operations */ +#include #include extern "C" { @@ -41,8 +42,6 @@ extern "C" { #include "src/core/lib/iomgr/exec_ctx.h" } -#include "third_party/benchmark/include/benchmark/benchmark.h" - static class InitializeStuff { public: InitializeStuff() { grpc_init(); } diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc index c63de0ce0ab..28ce170efac 100644 --- a/test/cpp/microbenchmarks/bm_fullstack.cc +++ b/test/cpp/microbenchmarks/bm_fullstack.cc @@ -35,6 +35,7 @@ #include +#include #include #include #include @@ -63,7 +64,6 @@ extern "C" { #include "src/core/lib/profiling/timers.h" #include "src/cpp/client/create_channel_internal.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" -#include "third_party/benchmark/include/benchmark/benchmark.h" namespace grpc { namespace testing { From b59b6b533fb83bb70cfb34f437e453b56dd5798f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Feb 2017 15:25:09 -0800 Subject: [PATCH 005/105] Add missing file --- third_party/benchmark.BUILD | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 third_party/benchmark.BUILD diff --git a/third_party/benchmark.BUILD b/third_party/benchmark.BUILD new file mode 100644 index 00000000000..4c622f32a84 --- /dev/null +++ b/third_party/benchmark.BUILD @@ -0,0 +1,15 @@ +cc_library( + name = "benchmark", + srcs = glob(["src/*.cc"]), + hdrs = glob(["include/**/*.h", "src/*.h"]), + includes = [ + "include", "." + ], + copts = [ + "-DHAVE_POSIX_REGEX" + ], + linkstatic = 1, + visibility = [ + "//visibility:public", + ], +) From b32ffb91459a57d6652d55d212d039d518e47b32 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Feb 2017 15:47:59 -0800 Subject: [PATCH 006/105] Fixes --- test/cpp/qps/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index 6f3265bdd43..9e68f8a0e99 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -84,6 +84,7 @@ cc_test( cc_binary( name = "qps_json_driver", srcs = ["qps_json_driver.cc"], + deps = ["//:grpc++", ":driver_impl"] ) cc_test( From 875b7a698455a9a9e2ddc6b6a05cb199fdc3c17c Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 10 Mar 2017 00:47:02 +0100 Subject: [PATCH 007/105] Fixing merge (?) --- Makefile | 23 +---------------------- build.yaml | 4 ---- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/Makefile b/Makefile index f6a50b2d513..9a347948fe2 100644 --- a/Makefile +++ b/Makefile @@ -12624,12 +12624,8 @@ endif endif -<<<<<<< HEAD $(BM_CALL_CREATE_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -======= $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_call_create.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a ->>>>>>> eb064ec7b81b60c5e1eb47d6124d0c05056b3097 deps_bm_call_create: $(BM_CALL_CREATE_OBJS:.o=.dep) @@ -12715,12 +12711,8 @@ endif endif -<<<<<<< HEAD $(BM_CLOSURE_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -======= $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_closure.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a ->>>>>>> eb064ec7b81b60c5e1eb47d6124d0c05056b3097 deps_bm_closure: $(BM_CLOSURE_OBJS:.o=.dep) @@ -12763,12 +12755,8 @@ endif endif -<<<<<<< HEAD $(BM_CQ_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -======= $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a ->>>>>>> eb064ec7b81b60c5e1eb47d6124d0c05056b3097 deps_bm_cq: $(BM_CQ_OBJS:.o=.dep) @@ -12811,12 +12799,7 @@ endif endif -<<<<<<< HEAD -$(BM_ERROR_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -======= $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a ->>>>>>> eb064ec7b81b60c5e1eb47d6124d0c05056b3097 deps_bm_error: $(BM_ERROR_OBJS:.o=.dep) @@ -12988,12 +12971,8 @@ endif endif -<<<<<<< HEAD -$(BM_FULLSTACK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX -$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o: $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -======= +$(BM_FULLSTACK_UNARY_PING_PONG_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a ->>>>>>> eb064ec7b81b60c5e1eb47d6124d0c05056b3097 deps_bm_fullstack_unary_ping_pong: $(BM_FULLSTACK_UNARY_PING_PONG_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index 8f7bd2de884..3abdeaefb22 100644 --- a/build.yaml +++ b/build.yaml @@ -3158,13 +3158,9 @@ targets: - gpr args: - --benchmark_min_time=0 -<<<<<<< HEAD - defaults: benchmark -======= excluded_poll_engines: - poll - poll-cv ->>>>>>> eb064ec7b81b60c5e1eb47d6124d0c05056b3097 platforms: - mac - linux From 0d033b51cedd2653e61ed7d24ebe748d4577d199 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Mar 2017 14:17:55 -0700 Subject: [PATCH 008/105] Test exposing TSAN race Isolated to ev_epoll_linux code Run with (on Linux): ``` tools/run_tests/run_tests.py -l c -c tsan -r ev_epoll -n inf -S --force_default_poller ``` --- test/core/iomgr/ev_epoll_linux_test.c | 85 +++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 4ec959995b2..b16b98028e6 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -43,6 +43,8 @@ #include #include +#include +#include #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/iomgr/workqueue.h" @@ -301,6 +303,88 @@ static void test_add_fd_to_pollset() { grpc_exec_ctx_finish(&exec_ctx); } +typedef struct threading_shared { + gpr_mu *mu; + grpc_pollset *pollset; + grpc_wakeup_fd *wakeup_fd; + grpc_fd *wakeup_desc; + grpc_closure on_wakeup; + int wakeups; +} threading_shared; + +static __thread bool thread_done = false; + +static void test_threading_loop(void *arg) { + threading_shared *shared = arg; + while (!thread_done) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_pollset_worker *worker; + gpr_mu_lock(shared->mu); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(&exec_ctx, shared->pollset, &worker, + gpr_now(GPR_CLOCK_MONOTONIC), + gpr_inf_future(GPR_CLOCK_MONOTONIC)))); + gpr_mu_unlock(shared->mu); + grpc_exec_ctx_finish(&exec_ctx); + } +} + +static void test_threading_wakeup(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + threading_shared *shared = arg; + if (++shared->wakeups > 1000000) { + thread_done = true; + } + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "consume_wakeup", grpc_wakeup_fd_consume_wakeup(shared->wakeup_fd))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_next", + grpc_wakeup_fd_wakeup(shared->wakeup_fd))); + grpc_fd_notify_on_read(exec_ctx, shared->wakeup_desc, &shared->on_wakeup); +} + +static void test_threading(void) { + threading_shared shared; + shared.pollset = gpr_zalloc(grpc_pollset_size()); + grpc_pollset_init(shared.pollset, &shared.mu); + + gpr_thd_id thds[100]; + for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { + gpr_thd_options opt = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&opt); + gpr_thd_new(&thds[i], test_threading_loop, &shared, &opt); + } + grpc_wakeup_fd fd; + GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&fd))); + shared.wakeup_fd = &fd; + shared.wakeup_desc = grpc_fd_create(fd.read_fd, "wakeup"); + shared.wakeups = 0; + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_pollset_add_fd(&exec_ctx, shared.pollset, shared.wakeup_desc); + grpc_fd_notify_on_read( + &exec_ctx, shared.wakeup_desc, + grpc_closure_init(&shared.on_wakeup, test_threading_wakeup, &shared, + grpc_schedule_on_exec_ctx)); + grpc_exec_ctx_finish(&exec_ctx); + } + GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_first", + grpc_wakeup_fd_wakeup(shared.wakeup_fd))); + for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { + gpr_thd_join(thds[i]); + } + fd.read_fd = 0; + grpc_wakeup_fd_destroy(&fd); + { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_fd_orphan(&exec_ctx, shared.wakeup_desc, NULL, NULL, "done"); + grpc_pollset_shutdown(&exec_ctx, shared.pollset, + grpc_closure_create(destroy_pollset, shared.pollset, + grpc_schedule_on_exec_ctx)); + grpc_exec_ctx_finish(&exec_ctx); + } +} + int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); @@ -310,6 +394,7 @@ int main(int argc, char **argv) { if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { test_add_fd_to_pollset(); test_pollset_queue_merge_items(); + test_threading(); } else { gpr_log(GPR_INFO, "Skipping the test. The test is only relevant for 'epoll' " From 78058b71aab5a77158275f2ac944dd749295501f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Mar 2017 15:01:56 -0700 Subject: [PATCH 009/105] Make repro more repro-able --- test/core/iomgr/ev_epoll_linux_test.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index b16b98028e6..0717a69cf6c 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -312,11 +312,11 @@ typedef struct threading_shared { int wakeups; } threading_shared; -static __thread bool thread_done = false; +static __thread int thread_wakeups = 0; static void test_threading_loop(void *arg) { threading_shared *shared = arg; - while (!thread_done) { + while (thread_wakeups < 1000000) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker; gpr_mu_lock(shared->mu); @@ -333,14 +333,13 @@ static void test_threading_loop(void *arg) { static void test_threading_wakeup(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { threading_shared *shared = arg; - if (++shared->wakeups > 1000000) { - thread_done = true; - } + ++shared->wakeups; + ++thread_wakeups; GPR_ASSERT(GRPC_LOG_IF_ERROR( "consume_wakeup", grpc_wakeup_fd_consume_wakeup(shared->wakeup_fd))); + grpc_fd_notify_on_read(exec_ctx, shared->wakeup_desc, &shared->on_wakeup); GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_next", grpc_wakeup_fd_wakeup(shared->wakeup_fd))); - grpc_fd_notify_on_read(exec_ctx, shared->wakeup_desc, &shared->on_wakeup); } static void test_threading(void) { @@ -348,7 +347,7 @@ static void test_threading(void) { shared.pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(shared.pollset, &shared.mu); - gpr_thd_id thds[100]; + gpr_thd_id thds[2]; for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); From ca3154d7c8edadcd84e1e199264c30a7f285058e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 17 Mar 2017 15:03:22 -0700 Subject: [PATCH 010/105] Make repro more repro-able --- test/core/iomgr/ev_epoll_linux_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 0717a69cf6c..6164c6dc4cd 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -347,7 +347,7 @@ static void test_threading(void) { shared.pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(shared.pollset, &shared.mu); - gpr_thd_id thds[2]; + gpr_thd_id thds[10]; for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); From d2164409343445e1f2de62b210218ad05c75d4c9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Mar 2017 06:45:39 -0700 Subject: [PATCH 011/105] Switch to no-barrier loads + full barrier cas-s to solve ABA problem --- include/grpc/impl/codegen/atm_gcc_atomic.h | 5 + include/grpc/impl/codegen/atm_gcc_sync.h | 1 + src/core/lib/iomgr/ev_epoll_linux.c | 115 ++++++++------------- 3 files changed, 47 insertions(+), 74 deletions(-) diff --git a/include/grpc/impl/codegen/atm_gcc_atomic.h b/include/grpc/impl/codegen/atm_gcc_atomic.h index 4bd3b257413..a486258c77f 100644 --- a/include/grpc/impl/codegen/atm_gcc_atomic.h +++ b/include/grpc/impl/codegen/atm_gcc_atomic.h @@ -85,6 +85,11 @@ static __inline int gpr_atm_rel_cas(gpr_atm *p, gpr_atm o, gpr_atm n) { p, &o, n, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED)); } +static __inline int gpr_atm_full_cas(gpr_atm *p, gpr_atm o, gpr_atm n) { + return GPR_ATM_INC_CAS_THEN(__atomic_compare_exchange_n( + p, &o, n, 0, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)); +} + #define gpr_atm_full_xchg(p, n) \ GPR_ATM_INC_CAS_THEN(__atomic_exchange_n((p), (n), __ATOMIC_ACQ_REL)) diff --git a/include/grpc/impl/codegen/atm_gcc_sync.h b/include/grpc/impl/codegen/atm_gcc_sync.h index 9aa2b43189f..946545a671d 100644 --- a/include/grpc/impl/codegen/atm_gcc_sync.h +++ b/include/grpc/impl/codegen/atm_gcc_sync.h @@ -83,6 +83,7 @@ static __inline void gpr_atm_no_barrier_store(gpr_atm *p, gpr_atm value) { #define gpr_atm_no_barrier_cas(p, o, n) gpr_atm_acq_cas((p), (o), (n)) #define gpr_atm_acq_cas(p, o, n) (__sync_bool_compare_and_swap((p), (o), (n))) #define gpr_atm_rel_cas(p, o, n) gpr_atm_acq_cas((p), (o), (n)) +#define gpr_atm_full_cas(p, o, n) gpr_atm_acq_cas((p), (o), (n)) static __inline gpr_atm gpr_atm_full_xchg(gpr_atm *p, gpr_atm n) { gpr_atm cur; diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 11208b9ad13..f1e14e9868f 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1107,19 +1107,15 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, grpc_closure *closure) { while (true) { - /* Fast-path: CLOSURE_NOT_READY -> . - The 'release' cas here matches the 'acquire' load in set_ready and - set_shutdown ensuring that the closure (scheduled by set_ready or - set_shutdown) happens-after the I/O event on the fd */ - if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { - return; /* Fast-path successful. Return */ - } - - /* Slowpath. The 'acquire' load matches the 'release' cas in set_ready and - set_shutdown */ - gpr_atm curr = gpr_atm_acq_load(state); + gpr_atm curr = gpr_atm_no_barrier_load(state); switch (curr) { case CLOSURE_NOT_READY: { + /* CLOSURE_NOT_READY -> . */ + if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, + (gpr_atm)closure)) { + return; /* Successful. Return */ + } + break; /* retry */ } @@ -1165,30 +1161,19 @@ static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, grpc_error *shutdown_err) { - /* Try the fast-path first (i.e expect the current value to be - CLOSURE_NOT_READY */ - gpr_atm curr = CLOSURE_NOT_READY; gpr_atm new_state = (gpr_atm)shutdown_err | FD_SHUTDOWN_BIT; while (true) { - /* The 'release' cas here matches the 'acquire' load in notify_on to ensure - that the closure it schedules 'happens-after' the set_shutdown is called - on the fd */ - if (gpr_atm_rel_cas(state, curr, new_state)) { - return; /* Fast-path successful. Return */ - } - /* Fallback to slowpath. This 'acquire' load matches the 'release' cas in notify_on and set_ready */ - curr = gpr_atm_acq_load(state); + gpr_atm curr = gpr_atm_no_barrier_load(state); switch (curr) { - case CLOSURE_READY: { - break; /* retry */ - } - - case CLOSURE_NOT_READY: { + case CLOSURE_READY: + case CLOSURE_NOT_READY: + if (gpr_atm_full_cas(state, curr, new_state)) { + return; /* early out */ + } break; /* retry */ - } default: { /* 'curr' is either a closure or the fd is already shutdown */ @@ -1199,10 +1184,8 @@ static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, } /* Fd is not shutdown. Schedule the closure and move the state to - shutdown state. The 'release' cas here matches the 'acquire' load in - notify_on to ensure that the closure it schedules 'happens-after' - the set_shutdown is called on the fd */ - if (gpr_atm_rel_cas(state, curr, new_state)) { + shutdown state. */ + if (gpr_atm_full_cas(state, curr, new_state)) { grpc_closure_sched( exec_ctx, (grpc_closure *)curr, GRPC_ERROR_CREATE_REFERENCING("FD Shutdown", &shutdown_err, 1)); @@ -1220,52 +1203,36 @@ static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, } static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state) { - /* Try an optimistic case first (i.e assume current state is - CLOSURE_NOT_READY). - - This 'release' cas matches the 'acquire' load in notify_on ensuring that - any closure (scheduled by notify_on) 'happens-after' the return from - epoll_pwait */ - if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, CLOSURE_READY)) { - return; /* early out */ - } - - /* The 'acquire' load here matches the 'release' cas in notify_on and - set_shutdown */ - gpr_atm curr = gpr_atm_acq_load(state); - switch (curr) { - case CLOSURE_READY: { - /* Already ready. We are done here */ - break; - } + while (true) { + gpr_atm curr = gpr_atm_acq_load(state); - case CLOSURE_NOT_READY: { - /* The state was not CLOSURE_NOT_READY when we checked initially at the - beginning of this function but now it is CLOSURE_NOT_READY again. - This is only possible if the state transitioned out of - CLOSURE_NOT_READY to either CLOSURE_READY or and then - back to CLOSURE_NOT_READY again (i.e after we entered this function, - the fd became "ready" and the necessary actions were already done). - So there is no need to make the state CLOSURE_READY now */ - break; - } + switch (curr) { + case CLOSURE_READY: { + /* Already ready. We are done here */ + return; + } - default: { - /* 'curr' is either a closure or the fd is shutdown */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - /* The fd is shutdown. Do nothing */ - } else if (gpr_atm_no_barrier_cas(state, curr, CLOSURE_NOT_READY)) { - /* The cas above was no-barrier since the state is being transitioned to - CLOSURE_NOT_READY; notify_on and set_shutdown do not schedule any - closures when transitioning out of CLOSURE_NO_READY state (i.e there - is no other code that needs to 'happen-after' this) */ + case CLOSURE_NOT_READY: { + if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, CLOSURE_READY)) { + return; /* early out */ + } + break; /* retry */ + } - grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); + default: { + /* 'curr' is either a closure or the fd is shutdown */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + /* The fd is shutdown. Do nothing */ + return; + } else if (gpr_atm_full_barrier_cas(state, curr, CLOSURE_NOT_READY)) { + grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); + return; + } + /* else the state changed again (only possible by either a racing + set_ready or set_shutdown functions. In both these cases, the closure + would have been scheduled for execution. So we are done here */ + return; } - /* else the state changed again (only possible by either a racing - set_ready or set_shutdown functions. In both these cases, the closure - would have been scheduled for execution. So we are done here */ - break; } } } From cad47dd47a51ca2bd264b47ac182a881f1fd4fac Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Mar 2017 06:59:09 -0700 Subject: [PATCH 012/105] Compile fix --- src/core/lib/iomgr/ev_epoll_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index f1e14e9868f..1e88fe76d3c 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1224,7 +1224,7 @@ static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state) { if ((curr & FD_SHUTDOWN_BIT) > 0) { /* The fd is shutdown. Do nothing */ return; - } else if (gpr_atm_full_barrier_cas(state, curr, CLOSURE_NOT_READY)) { + } else if (gpr_atm_full_cas(state, curr, CLOSURE_NOT_READY)) { grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); return; } From 2b4a040d406efb6c27a4ca3732a46145b8ce7d9f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Mar 2017 09:21:20 -0700 Subject: [PATCH 013/105] Fix race? --- src/core/lib/iomgr/ev_epoll_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 1e88fe76d3c..3b35491df9f 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1111,7 +1111,7 @@ static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, switch (curr) { case CLOSURE_NOT_READY: { /* CLOSURE_NOT_READY -> . */ - if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, + if (gpr_atm_full_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { return; /* Successful. Return */ } From b2d8e03c5e1928464efc8251f9406cf8506f8973 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 21 Mar 2017 11:01:48 -0700 Subject: [PATCH 014/105] Weaken some barriers that were stronger than necessary --- src/core/lib/iomgr/ev_epoll_linux.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 3b35491df9f..d5acdf0d4b4 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1111,8 +1111,7 @@ static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, switch (curr) { case CLOSURE_NOT_READY: { /* CLOSURE_NOT_READY -> . */ - if (gpr_atm_full_cas(state, CLOSURE_NOT_READY, - (gpr_atm)closure)) { + if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { return; /* Successful. Return */ } @@ -1204,7 +1203,7 @@ static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state) { while (true) { - gpr_atm curr = gpr_atm_acq_load(state); + gpr_atm curr = gpr_atm_no_barrier_load(state); switch (curr) { case CLOSURE_READY: { From 0a6d30bc4c4a92fc946a69f8c2205c445fdafc59 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 22 Mar 2017 00:51:51 +0100 Subject: [PATCH 015/105] Buildifier. --- test/cpp/qps/BUILD | 96 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 24 deletions(-) diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index 9e68f8a0e99..d818643bc32 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -31,83 +31,131 @@ licenses(["notice"]) # 3-clause BSD cc_library( name = "parse_json", - hdrs = ["parse_json.h"], srcs = ["parse_json.cc"], + hdrs = ["parse_json.h"], deps = ["//:grpc++"], ) cc_library( name = "qps_worker_impl", - hdrs = ["client.h", "server.h", "qps_worker.h"], - srcs = ["client_async.cc", "client_sync.cc", "server_async.cc", "server_sync.cc", "qps_worker.cc"], - deps = ["//:grpc++", - "//:grpc", - "//test/core/util:grpc_test_util", - "//test/core/util:gpr_test_util", - "//src/proto/grpc/testing:services_proto", - "//src/proto/grpc/testing:payloads_proto", - "//src/proto/grpc/testing:control_proto", - "//external:gtest"], + srcs = [ + "client_async.cc", + "client_sync.cc", + "qps_worker.cc", + "server_async.cc", + "server_sync.cc", + ], + hdrs = [ + "client.h", + "qps_worker.h", + "server.h", + ], + deps = [ + "//:grpc", + "//:grpc++", + "//external:gtest", + "//src/proto/grpc/testing:control_proto", + "//src/proto/grpc/testing:payloads_proto", + "//src/proto/grpc/testing:services_proto", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], ) cc_library( name = "driver_impl", - hdrs = ["driver.h", "report.h"], - srcs = ["driver.cc", "report.cc"], - deps = ["//:grpc++"] + srcs = [ + "driver.cc", + "report.cc", + ], + hdrs = [ + "driver.h", + "report.h", + ], + deps = [ + "//:grpc++", + "//src/proto/grpc/testing:control_proto", + ], ) cc_library( name = "histogram", - hdrs = ["histogram.h", "stats.h"], - deps = ["//:gpr"] + hdrs = [ + "histogram.h", + "stats.h", + ], + deps = ["//:gpr"], ) cc_library( name = "interarrival", hdrs = ["interarrival.h"], - deps = ["//:grpc++"] + deps = ["//:grpc++"], ) cc_binary( name = "json_run_localhost", srcs = ["json_run_localhost.cc"], - deps = ["//:gpr", "//test/core/util:grpc_test_util", "//test/core/util:gpr_test_util", "//test/cpp/util:test_util"] + deps = [ + "//:gpr", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + "//test/cpp/util:test_util", + ], ) cc_test( name = "qps_interarrival_test", srcs = ["qps_interarrival_test.cc"], - deps = [":histogram", ":interarrival"] + deps = [ + ":histogram", + ":interarrival", + ], ) cc_binary( name = "qps_json_driver", srcs = ["qps_json_driver.cc"], - deps = ["//:grpc++", ":driver_impl"] + deps = [ + ":driver_impl", + "//:grpc++", + ], ) cc_test( name = "qps_openloop_test", srcs = ["qps_openloop_test.cc"], - deps = [":qps_worker_impl", ":driver_impl"] + deps = [ + ":driver_impl", + ":qps_worker_impl", + ], ) cc_test( name = "secure_sync_unary_ping_pong_test", srcs = ["secure_sync_unary_ping_pong_test.cc"], - deps = ["//:grpc++", ":driver_impl"] + deps = [ + ":driver_impl", + "//:grpc++", + ], ) cc_library( name = "usage_timer", srcs = ["usage_timer.cc"], hdrs = ["usage_timer.h"], - deps = ["//:gpr"] + deps = ["//:gpr"], ) cc_binary( name = "qps_worker", srcs = ["worker.cc"], - deps = ["//:grpc++", ":qps_worker_impl", "//test/core/util:grpc_test_util", "//test/core/util:gpr_test_util", "//test/cpp/util:test_util", "//test/cpp/util:test_config"] + deps = [ + ":qps_worker_impl", + "//:grpc++", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + "//test/cpp/util:test_config", + "//test/cpp/util:test_util", + ], ) From b02dd01584e21565ebea173ceee7568404d23f2b Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Fri, 24 Mar 2017 18:45:22 +0100 Subject: [PATCH 016/105] Fixing qps & benchmark. --- CMakeLists.txt | 2 +- Makefile | 6 ++-- build.yaml | 4 +-- test/cpp/microbenchmarks/BUILD | 2 ++ test/cpp/qps/BUILD | 33 +++++++++++++++++++ test/cpp/{util => qps}/benchmark_config.cc | 2 +- test/cpp/{util => qps}/benchmark_config.h | 0 test/cpp/qps/qps_json_driver.cc | 2 +- test/cpp/qps/qps_openloop_test.cc | 2 +- test/cpp/qps/qps_test.cc | 2 +- .../qps/secure_sync_unary_ping_pong_test.cc | 2 +- .../generated/sources_and_headers.json | 10 +++--- vsprojects/vcxproj/qps/qps.vcxproj | 6 ++-- vsprojects/vcxproj/qps/qps.vcxproj.filters | 15 ++++----- 14 files changed, 60 insertions(+), 28 deletions(-) rename test/cpp/{util => qps}/benchmark_config.cc (98%) rename test/cpp/{util => qps}/benchmark_config.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a8bff55342..57d7d1adf60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3660,6 +3660,7 @@ add_library(qps ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/services.grpc.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/services.pb.h ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/services.grpc.pb.h + test/cpp/qps/benchmark_config.cc test/cpp/qps/client_async.cc test/cpp/qps/client_sync.cc test/cpp/qps/driver.cc @@ -3669,7 +3670,6 @@ add_library(qps test/cpp/qps/server_async.cc test/cpp/qps/server_sync.cc test/cpp/qps/usage_timer.cc - test/cpp/util/benchmark_config.cc ) if(WIN32 AND MSVC) diff --git a/Makefile b/Makefile index 587a9c04cd3..1a5d084ac70 100644 --- a/Makefile +++ b/Makefile @@ -5494,6 +5494,7 @@ LIBQPS_SRC = \ $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc \ $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc \ $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc \ + test/cpp/qps/benchmark_config.cc \ test/cpp/qps/client_async.cc \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ @@ -5503,7 +5504,6 @@ LIBQPS_SRC = \ test/cpp/qps/server_async.cc \ test/cpp/qps/server_sync.cc \ test/cpp/qps/usage_timer.cc \ - test/cpp/util/benchmark_config.cc \ PUBLIC_HEADERS_CXX += \ @@ -5549,6 +5549,7 @@ ifneq ($(NO_DEPS),true) -include $(LIBQPS_OBJS:.o=.dep) endif endif +$(OBJDIR)/$(CONFIG)/test/cpp/qps/benchmark_config.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc @@ -5558,7 +5559,6 @@ $(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/src/proto/grpc/testing/mess $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/usage_timer.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.pb.cc $(GENDIR)/src/proto/grpc/testing/payloads.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/services.pb.cc $(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc LIBGRPC_CSHARP_EXT_SRC = \ @@ -18805,6 +18805,7 @@ test/cpp/interop/interop_server.cc: $(OPENSSL_DEP) test/cpp/interop/interop_server_bootstrap.cc: $(OPENSSL_DEP) test/cpp/interop/server_helper.cc: $(OPENSSL_DEP) test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP) +test/cpp/qps/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/qps/client_async.cc: $(OPENSSL_DEP) test/cpp/qps/client_sync.cc: $(OPENSSL_DEP) test/cpp/qps/driver.cc: $(OPENSSL_DEP) @@ -18814,7 +18815,6 @@ test/cpp/qps/report.cc: $(OPENSSL_DEP) test/cpp/qps/server_async.cc: $(OPENSSL_DEP) test/cpp/qps/server_sync.cc: $(OPENSSL_DEP) test/cpp/qps/usage_timer.cc: $(OPENSSL_DEP) -test/cpp/util/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/util/byte_buffer_proto_helper.cc: $(OPENSSL_DEP) test/cpp/util/cli_call.cc: $(OPENSSL_DEP) test/cpp/util/cli_credentials.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 994aeea71b5..17292565897 100644 --- a/build.yaml +++ b/build.yaml @@ -1384,6 +1384,7 @@ libs: build: private language: c++ headers: + - test/cpp/qps/benchmark_config.h - test/cpp/qps/client.h - test/cpp/qps/driver.h - test/cpp/qps/histogram.h @@ -1394,13 +1395,13 @@ libs: - test/cpp/qps/server.h - test/cpp/qps/stats.h - test/cpp/qps/usage_timer.h - - test/cpp/util/benchmark_config.h src: - src/proto/grpc/testing/messages.proto - src/proto/grpc/testing/payloads.proto - src/proto/grpc/testing/stats.proto - src/proto/grpc/testing/control.proto - src/proto/grpc/testing/services.proto + - test/cpp/qps/benchmark_config.cc - test/cpp/qps/client_async.cc - test/cpp/qps/client_sync.cc - test/cpp/qps/driver.cc @@ -1410,7 +1411,6 @@ libs: - test/cpp/qps/server_async.cc - test/cpp/qps/server_sync.cc - test/cpp/qps/usage_timer.cc - - test/cpp/util/benchmark_config.cc deps: - grpc_test_util - grpc++_test_util diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 8f8858367f8..38619666dcb 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -33,6 +33,7 @@ cc_test( name = "noop-benchmark", srcs = ["noop-benchmark.cc"], deps = ["//external:benchmark"], + linkopts = ["-pthread"], ) cc_library( @@ -40,6 +41,7 @@ cc_library( srcs = ["helpers.cc"], hdrs = ["helpers.h", "fullstack_fixtures.h", "fullstack_context_mutators.h"], deps = ["//:grpc++", "//external:benchmark", "//test/core/util:grpc_test_util", "//src/proto/grpc/testing:echo_proto"], + linkopts = ["-pthread"], ) cc_test( diff --git a/test/cpp/qps/BUILD b/test/cpp/qps/BUILD index d818643bc32..6492b63ec30 100644 --- a/test/cpp/qps/BUILD +++ b/test/cpp/qps/BUILD @@ -51,14 +51,19 @@ cc_library( "server.h", ], deps = [ + ":histogram", + ":interarrival", + ":usage_timer", "//:grpc", "//:grpc++", "//external:gtest", "//src/proto/grpc/testing:control_proto", "//src/proto/grpc/testing:payloads_proto", "//src/proto/grpc/testing:services_proto", + "//test/core/end2end:ssl_test_data", "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", + "//test/cpp/util:test_util", ], ) @@ -73,7 +78,31 @@ cc_library( "report.h", ], deps = [ + ":histogram", + ":parse_json", + ":qps_worker_impl", + "//:grpc++", + "//src/proto/grpc/testing:control_proto", + "//src/proto/grpc/testing:messages_proto", + "//src/proto/grpc/testing:services_proto", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + +cc_library( + name = "benchmark_config", + srcs = [ + "benchmark_config.cc", + ], + hdrs = [ + "benchmark_config.h", + ], + deps = [ + ":driver_impl", + ":histogram", "//:grpc++", + "//external:gflags", "//src/proto/grpc/testing:control_proto", ], ) @@ -117,8 +146,10 @@ cc_binary( name = "qps_json_driver", srcs = ["qps_json_driver.cc"], deps = [ + ":benchmark_config", ":driver_impl", "//:grpc++", + "//external:gflags", ], ) @@ -126,6 +157,7 @@ cc_test( name = "qps_openloop_test", srcs = ["qps_openloop_test.cc"], deps = [ + ":benchmark_config", ":driver_impl", ":qps_worker_impl", ], @@ -135,6 +167,7 @@ cc_test( name = "secure_sync_unary_ping_pong_test", srcs = ["secure_sync_unary_ping_pong_test.cc"], deps = [ + ":benchmark_config", ":driver_impl", "//:grpc++", ], diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/qps/benchmark_config.cc similarity index 98% rename from test/cpp/util/benchmark_config.cc rename to test/cpp/qps/benchmark_config.cc index 6fc864069ef..98b8d0ba379 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/qps/benchmark_config.cc @@ -31,7 +31,7 @@ * */ -#include "test/cpp/util/benchmark_config.h" +#include "test/cpp/qps/benchmark_config.h" #include DEFINE_bool(enable_log_reporter, true, diff --git a/test/cpp/util/benchmark_config.h b/test/cpp/qps/benchmark_config.h similarity index 100% rename from test/cpp/util/benchmark_config.h rename to test/cpp/qps/benchmark_config.h diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc index bd2c1f0ec67..53df4abcd38 100644 --- a/test/cpp/qps/qps_json_driver.cc +++ b/test/cpp/qps/qps_json_driver.cc @@ -43,7 +43,7 @@ #include "test/cpp/qps/driver.h" #include "test/cpp/qps/parse_json.h" #include "test/cpp/qps/report.h" -#include "test/cpp/util/benchmark_config.h" +#include "test/cpp/qps/benchmark_config.h" DEFINE_string(scenarios_file, "", "JSON file containing an array of Scenario objects"); diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc index 70e2709ac02..7f820a39142 100644 --- a/test/cpp/qps/qps_openloop_test.cc +++ b/test/cpp/qps/qps_openloop_test.cc @@ -38,7 +38,7 @@ #include "test/core/util/test_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" -#include "test/cpp/util/benchmark_config.h" +#include "test/cpp/qps/benchmark_config.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc index f94ea0cb499..d5a6ff621d0 100644 --- a/test/cpp/qps/qps_test.cc +++ b/test/cpp/qps/qps_test.cc @@ -37,7 +37,7 @@ #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" -#include "test/cpp/util/benchmark_config.h" +#include "test/cpp/qps/benchmark_config.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc index d0c47d102ad..8afcebb6cf4 100644 --- a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc +++ b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc @@ -37,7 +37,7 @@ #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" -#include "test/cpp/util/benchmark_config.h" +#include "test/cpp/qps/benchmark_config.h" namespace grpc { namespace testing { diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 14ad133863a..7e41be0f34e 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -6188,6 +6188,7 @@ "src/proto/grpc/testing/services.pb.h", "src/proto/grpc/testing/stats.grpc.pb.h", "src/proto/grpc/testing/stats.pb.h", + "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/driver.h", "test/cpp/qps/histogram.h", @@ -6197,13 +6198,14 @@ "test/cpp/qps/report.h", "test/cpp/qps/server.h", "test/cpp/qps/stats.h", - "test/cpp/qps/usage_timer.h", - "test/cpp/util/benchmark_config.h" + "test/cpp/qps/usage_timer.h" ], "is_filegroup": false, "language": "c++", "name": "qps", "src": [ + "test/cpp/qps/benchmark_config.cc", + "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/client_async.cc", "test/cpp/qps/client_sync.cc", @@ -6222,9 +6224,7 @@ "test/cpp/qps/server_sync.cc", "test/cpp/qps/stats.h", "test/cpp/qps/usage_timer.cc", - "test/cpp/qps/usage_timer.h", - "test/cpp/util/benchmark_config.cc", - "test/cpp/util/benchmark_config.h" + "test/cpp/qps/usage_timer.h" ], "third_party": false, "type": "lib" diff --git a/vsprojects/vcxproj/qps/qps.vcxproj b/vsprojects/vcxproj/qps/qps.vcxproj index 2d7892683a8..6e290f4557d 100644 --- a/vsprojects/vcxproj/qps/qps.vcxproj +++ b/vsprojects/vcxproj/qps/qps.vcxproj @@ -147,6 +147,7 @@ + @@ -157,7 +158,6 @@ - @@ -200,6 +200,8 @@ + + @@ -218,8 +220,6 @@ - - diff --git a/vsprojects/vcxproj/qps/qps.vcxproj.filters b/vsprojects/vcxproj/qps/qps.vcxproj.filters index f25be9af191..a7d6c127e8a 100644 --- a/vsprojects/vcxproj/qps/qps.vcxproj.filters +++ b/vsprojects/vcxproj/qps/qps.vcxproj.filters @@ -16,6 +16,9 @@ src\proto\grpc\testing + + test\cpp\qps + test\cpp\qps @@ -43,11 +46,11 @@ test\cpp\qps - - test\cpp\util - + + test\cpp\qps + test\cpp\qps @@ -78,9 +81,6 @@ test\cpp\qps - - test\cpp\util - @@ -105,9 +105,6 @@ {b57fa0e4-f88d-fe46-8885-956fc582de3d} - - {9042d134-6d5a-a907-799e-01768a475055} - From e0f894087777e03474d6e35fa6767bcc44343a2b Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Fri, 24 Mar 2017 14:26:55 -0700 Subject: [PATCH 017/105] update go version in Dockerfile to 1.8 --- .../dockerfile/interoptest/grpc_interop_go/Dockerfile.template | 2 +- .../interoptest/grpc_interop_http2/Dockerfile.template | 2 +- .../stress_test/grpc_interop_stress_go/Dockerfile.template | 2 +- tools/dockerfile/interoptest/grpc_interop_go/Dockerfile | 2 +- tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template index 38a5ca725da..bf025d80371 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 12b99844253..a49e9512892 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template index e02254cd535..ad8ad71b5fa 100644 --- a/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index d3bf071c72e..d7a0b1786b9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index acc984acb0d..da001528254 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile index bbf7de7f91d..c099f339aee 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean From 34d21ceadcc7e83f15f3bdcfa6955940c6e4d30a Mon Sep 17 00:00:00 2001 From: thinkerou Date: Sun, 26 Mar 2017 01:07:27 +0800 Subject: [PATCH 018/105] fix php style --- src/php/lib/Grpc/AbstractCall.php | 2 +- src/php/tests/unit_tests/ServerTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/php/lib/Grpc/AbstractCall.php b/src/php/lib/Grpc/AbstractCall.php index 4833fdc7b6f..a59bfa3ba38 100644 --- a/src/php/lib/Grpc/AbstractCall.php +++ b/src/php/lib/Grpc/AbstractCall.php @@ -131,7 +131,7 @@ abstract class AbstractCall // Proto3 implementation if (method_exists($data, 'encode')) { return $data->encode(); - } else if (method_exists($data, 'serializeToString')) { + } elseif (method_exists($data, 'serializeToString')) { return $data->serializeToString(); } diff --git a/src/php/tests/unit_tests/ServerTest.php b/src/php/tests/unit_tests/ServerTest.php index 5f40202f182..3e7c01f20e2 100644 --- a/src/php/tests/unit_tests/ServerTest.php +++ b/src/php/tests/unit_tests/ServerTest.php @@ -69,7 +69,7 @@ class ServerTest extends PHPUnit_Framework_TestCase $this->server = new Grpc\Server(); $port = $this->server->addHttp2Port('0.0.0.0:0'); $this->server->start(); - $channel = new Grpc\Channel('localhost:' . $port, + $channel = new Grpc\Channel('localhost:'.$port, ['credentials' => Grpc\ChannelCredentials::createInsecure()]); $deadline = Grpc\Timeval::infFuture(); From 0fe74d35ee31ac3fb38fe647786b0b71629d64d0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 26 Mar 2017 21:25:04 -0700 Subject: [PATCH 019/105] Initial historical line counter --- tools/line_count/collect-history.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 tools/line_count/collect-history.py diff --git a/tools/line_count/collect-history.py b/tools/line_count/collect-history.py new file mode 100755 index 00000000000..0de95f2bac7 --- /dev/null +++ b/tools/line_count/collect-history.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +import subprocess +import datetime + +def daterange(start, end): + for n in range(int((end - start).days)): + yield start + datetime.timedelta(n) + +start_date = datetime.date(2014, 11, 21) +end_date = datetime.date(2017, 3, 26) + +for dt in daterange(start_date, end_date): + dmy = dt.strftime('%Y-%m-%d') + sha1 = subprocess.check_output(['git', 'rev-list', '-n', '1', + '--before=%s' % dmy, + 'master']).strip() + subprocess.check_call(['git', 'checkout', sha1]) + subprocess.check_call(['git', 'submodule', 'update']) + subprocess.check_call(['git', 'clean', '-f', '-x', '-d']) + subprocess.check_call(['cloc', '--vcs=git', '--by-file', '--yaml', '--out=../count/%s.yaml' % dmy, '.']) + From 6f7dcd0ade302ab348832bf3e765198be34278f9 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Mon, 27 Mar 2017 18:51:55 +0200 Subject: [PATCH 020/105] Clang-format. --- test/cpp/qps/qps_json_driver.cc | 2 +- test/cpp/qps/qps_openloop_test.cc | 2 +- test/cpp/qps/qps_test.cc | 2 +- test/cpp/qps/secure_sync_unary_ping_pong_test.cc | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc index 53df4abcd38..a9061374748 100644 --- a/test/cpp/qps/qps_json_driver.cc +++ b/test/cpp/qps/qps_json_driver.cc @@ -40,10 +40,10 @@ #include #include +#include "test/cpp/qps/benchmark_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/parse_json.h" #include "test/cpp/qps/report.h" -#include "test/cpp/qps/benchmark_config.h" DEFINE_string(scenarios_file, "", "JSON file containing an array of Scenario objects"); diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc index 7f820a39142..28b396739fb 100644 --- a/test/cpp/qps/qps_openloop_test.cc +++ b/test/cpp/qps/qps_openloop_test.cc @@ -36,9 +36,9 @@ #include #include "test/core/util/test_config.h" +#include "test/cpp/qps/benchmark_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" -#include "test/cpp/qps/benchmark_config.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc index d5a6ff621d0..7c4e2cfd3e1 100644 --- a/test/cpp/qps/qps_test.cc +++ b/test/cpp/qps/qps_test.cc @@ -35,9 +35,9 @@ #include +#include "test/cpp/qps/benchmark_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" -#include "test/cpp/qps/benchmark_config.h" namespace grpc { namespace testing { diff --git a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc index 8afcebb6cf4..509d9f89c38 100644 --- a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc +++ b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc @@ -35,9 +35,9 @@ #include +#include "test/cpp/qps/benchmark_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/report.h" -#include "test/cpp/qps/benchmark_config.h" namespace grpc { namespace testing { From 0d04e0697b9e8647e6bd0d2a9c17681b57865058 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 14:55:08 -0700 Subject: [PATCH 021/105] Add a benchmark of base16 encoding --- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 158 +++++++++++++++++++- 1 file changed, 155 insertions(+), 3 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 563db758f71..b61b325dfa9 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -33,6 +33,7 @@ /* Microbenchmarks around CHTTP2 HPACK operations */ +#include #include #include #include @@ -40,6 +41,7 @@ extern "C" { #include "src/core/ext/transport/chttp2/transport/hpack_encoder.h" #include "src/core/ext/transport/chttp2/transport/hpack_parser.h" #include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/static_metadata.h" } #include "test/cpp/microbenchmarks/helpers.h" @@ -69,6 +71,7 @@ template static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { TrackCounters track_counters; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + static bool logged_representative_output = false; grpc_metadata_batch b; grpc_metadata_batch_init(&b); @@ -87,8 +90,17 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State &state) { grpc_slice_buffer outbuf; grpc_slice_buffer_init(&outbuf); while (state.KeepRunning()) { - grpc_chttp2_encode_header(&exec_ctx, &c, (uint32_t)state.iterations(), &b, - state.range(0), state.range(1), &stats, &outbuf); + uint32_t stream_id = static_cast(state.iterations()); + grpc_chttp2_encode_header(&exec_ctx, &c, stream_id, &b, state.range(0), + state.range(1), &stats, &outbuf); + if (!logged_representative_output) { + logged_representative_output = true; + for (size_t i = 0; i < outbuf.count; i++) { + char *s = grpc_dump_slice(outbuf.slices[i], GPR_DUMP_HEX); + gpr_log(GPR_DEBUG, "%" PRId64 ": %s", i, s); + gpr_free(s); + } + } grpc_slice_buffer_reset_and_unref_internal(&exec_ctx, &outbuf); grpc_exec_ctx_flush(&exec_ctx); } @@ -131,6 +143,25 @@ class SingleInternedElem { } }; +template +class SingleInternedBinaryElem { + public: + static std::vector GetElems(grpc_exec_ctx *exec_ctx) { + return {grpc_mdelem_from_slices( + exec_ctx, grpc_slice_intern(grpc_slice_from_static_string("abc-bin")), + grpc_slice_intern(MakeBytes()))}; + } + + private: + static grpc_slice MakeBytes() { + std::vector v; + for (int i = 0; i < kLength; i++) { + v.push_back(static_cast(rand())); + } + return grpc_slice_from_copied_buffer(v.data(), v.size()); + } +}; + class SingleInternedKeyElem { public: static std::vector GetElems(grpc_exec_ctx *exec_ctx) { @@ -149,6 +180,24 @@ class SingleNonInternedElem { } }; +template +class SingleNonInternedBinaryElem { + public: + static std::vector GetElems(grpc_exec_ctx *exec_ctx) { + return {grpc_mdelem_from_slices( + exec_ctx, grpc_slice_from_static_string("abc-bin"), MakeBytes())}; + } + + private: + static grpc_slice MakeBytes() { + std::vector v; + for (int i = 0; i < kLength; i++) { + v.push_back(static_cast(rand())); + } + return grpc_slice_from_copied_buffer(v.data(), v.size()); + } +}; + class RepresentativeClientInitialMetadata { public: static std::vector GetElems(grpc_exec_ctx *exec_ctx) { @@ -195,8 +244,29 @@ BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedKeyElem) ->Args({0, 16384}); BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedElem) ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedBinaryElem<1>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedBinaryElem<3>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedBinaryElem<10>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedBinaryElem<31>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleInternedBinaryElem<100>) + ->Args({0, 16384}); BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleNonInternedElem) ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleNonInternedBinaryElem<1>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleNonInternedBinaryElem<3>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleNonInternedBinaryElem<10>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleNonInternedBinaryElem<31>) + ->Args({0, 16384}); +BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, + SingleNonInternedBinaryElem<100>) + ->Args({0, 16384}); // test with a tiny frame size, to highlight continuation costs BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader, SingleNonInternedElem) ->Args({0, 1}); @@ -262,7 +332,7 @@ static void BM_HpackParserParseHeader(benchmark::State &state) { namespace hpack_parser_fixtures { -static grpc_slice MakeSlice(std::initializer_list bytes) { +static grpc_slice MakeSlice(std::vector bytes) { grpc_slice s = grpc_slice_malloc(bytes.size()); uint8_t *p = GRPC_SLICE_START_PTR(s); for (auto b : bytes) { @@ -346,6 +416,64 @@ class NonIndexedElem { } }; +class NonIndexedBinaryElem1 { + public: + static std::vector GetInitSlices() { return {}; } + static std::vector GetBenchmarkSlices() { + return {MakeSlice( + {0x00, 0x07, 'a', 'b', 'c', '-', 'b', 'i', 'n', 0x82, 0xf7, 0xb3})}; + } +}; + +class NonIndexedBinaryElem3 { + public: + static std::vector GetInitSlices() { return {}; } + static std::vector GetBenchmarkSlices() { + return {MakeSlice({0x00, 0x07, 'a', 'b', 'c', '-', 'b', 'i', 'n', 0x84, + 0x7f, 0x4e, 0x29, 0x3f})}; + } +}; + +class NonIndexedBinaryElem10 { + public: + static std::vector GetInitSlices() { return {}; } + static std::vector GetBenchmarkSlices() { + return {MakeSlice({0x00, 0x07, 'a', 'b', 'c', '-', 'b', + 'i', 'n', 0x8b, 0x71, 0x0c, 0xa5, 0x81, + 0x73, 0x7b, 0x47, 0x13, 0xe9, 0xf7, 0xe3})}; + } +}; + +class NonIndexedBinaryElem31 { + public: + static std::vector GetInitSlices() { return {}; } + static std::vector GetBenchmarkSlices() { + return {MakeSlice({0x00, 0x07, 'a', 'b', 'c', '-', 'b', 'i', 'n', + 0xa3, 0x92, 0x43, 0x7f, 0xbe, 0x7c, 0xea, 0x6f, 0xf3, + 0x3d, 0xa7, 0xa7, 0x67, 0xfb, 0xe2, 0x82, 0xf7, 0xf2, + 0x8f, 0x1f, 0x9d, 0xdf, 0xf1, 0x7e, 0xb3, 0xef, 0xb2, + 0x8f, 0x53, 0x77, 0xce, 0x0c, 0x13, 0xe3, 0xfd, 0x87})}; + } +}; + +class NonIndexedBinaryElem100 { + public: + static std::vector GetInitSlices() { return {}; } + static std::vector GetBenchmarkSlices() { + return {MakeSlice( + {0x00, 0x07, 'a', 'b', 'c', '-', 'b', 'i', 'n', 0xeb, 0x1d, 0x4d, + 0xe8, 0x96, 0x8c, 0x14, 0x20, 0x06, 0xc1, 0xc3, 0xdf, 0x6e, 0x1f, 0xef, + 0xde, 0x2f, 0xde, 0xb7, 0xf2, 0xfe, 0x6d, 0xd4, 0xe4, 0x7d, 0xf5, 0x55, + 0x46, 0x52, 0x3d, 0x91, 0xf2, 0xd4, 0x6f, 0xca, 0x34, 0xcd, 0xd9, 0x39, + 0xbd, 0x03, 0x27, 0xe3, 0x9c, 0x74, 0xcc, 0x17, 0x34, 0xed, 0xa6, 0x6a, + 0x77, 0x73, 0x10, 0xcd, 0x8e, 0x4e, 0x5c, 0x7c, 0x72, 0x39, 0xd8, 0xe6, + 0x78, 0x6b, 0xdb, 0xa5, 0xb7, 0xab, 0xe7, 0x46, 0xae, 0x21, 0xab, 0x7f, + 0x01, 0x89, 0x13, 0xd7, 0xca, 0x17, 0x6e, 0xcb, 0xd6, 0x79, 0x71, 0x68, + 0xbf, 0x8a, 0x3f, 0x32, 0xe8, 0xba, 0xf5, 0xbe, 0xb3, 0xbc, 0xde, 0x28, + 0xc7, 0xcf, 0x62, 0x7a, 0x58, 0x2c, 0xcf, 0x4d, 0xe3})}; + } +}; + class RepresentativeClientInitialMetadata { public: static std::vector GetInitSlices() { @@ -437,6 +565,11 @@ BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, IndexedSingleInternedElem); BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, AddIndexedSingleInternedElem); BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, KeyIndexedSingleInternedElem); BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, NonIndexedElem); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, NonIndexedBinaryElem1); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, NonIndexedBinaryElem3); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, NonIndexedBinaryElem10); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, NonIndexedBinaryElem31); +BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, NonIndexedBinaryElem100); BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, RepresentativeClientInitialMetadata); BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, @@ -446,4 +579,23 @@ BENCHMARK_TEMPLATE(BM_HpackParserParseHeader, } // namespace hpack_parser_fixtures +static void BM_Base16SomeStuff(benchmark::State &state) { + uint8_t *bytes = new uint8_t[state.range(0)]; + for (int i = 0; i < state.range(0); i++) { + bytes[i] = static_cast(rand()); + } + uint8_t *encoded = new uint8_t[state.range(0) * 2]; + static const uint8_t hex[] = "0123456789abcdef"; + while (state.KeepRunning()) { + for (int i = 0; i < state.range(0); i++) { + encoded[2 * i + 0] = hex[encoded[i] >> 8]; + encoded[2 * i + 1] = hex[encoded[i] & 0xf]; + } + } + delete[] encoded; + delete[] bytes; + state.SetBytesProcessed(state.iterations() * state.range(0)); +} +BENCHMARK(BM_Base16SomeStuff)->Range(1, 4096); + BENCHMARK_MAIN(); From 0ff932cf080ba0f73bec20919dfd295a144fd75c Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 27 Mar 2017 16:16:52 -0700 Subject: [PATCH 022/105] Make sure that we hold contents of async operation until completion. --- test/cpp/qps/server_async.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index b58d91eea63..b499b82091e 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -235,18 +235,17 @@ class AsyncQpsServerTest final : public grpc::testing::Server { return false; } - ResponseType response; - // Call the RPC processing function - grpc::Status status = invoke_method_(&req_, &response); + grpc::Status status = invoke_method_(&req_, &response_); // Have the response writer work and invoke on_finish when done next_state_ = &ServerRpcContextUnaryImpl::finisher; - response_writer_.Finish(response, status, AsyncQpsServerTest::tag(this)); + response_writer_.Finish(response_, status, AsyncQpsServerTest::tag(this)); return true; } std::unique_ptr srv_ctx_; RequestType req_; + ResponseType response_; bool (ServerRpcContextUnaryImpl::*next_state_)(bool); std::function *, void *)> @@ -298,11 +297,10 @@ class AsyncQpsServerTest final : public grpc::testing::Server { bool read_done(bool ok) { if (ok) { // invoke the method - ResponseType response; // Call the RPC processing function - grpc::Status status = invoke_method_(&req_, &response); + grpc::Status status = invoke_method_(&req_, &response_); // initiate the write - stream_.Write(response, AsyncQpsServerTest::tag(this)); + stream_.Write(response_, AsyncQpsServerTest::tag(this)); next_state_ = &ServerRpcContextStreamingImpl::write_done; } else { // client has sent writes done // finish the stream @@ -326,6 +324,7 @@ class AsyncQpsServerTest final : public grpc::testing::Server { std::unique_ptr srv_ctx_; RequestType req_; + ResponseType response_; bool (ServerRpcContextStreamingImpl::*next_state_)(bool); std::function Date: Mon, 27 Mar 2017 21:25:15 -0700 Subject: [PATCH 023/105] A more useful bm_diff? --- tools/profiling/microbenchmarks/bm_diff.py | 181 ++++++++++++++------- tools/run_tests/run_microbenchmark.py | 32 ---- 2 files changed, 120 insertions(+), 93 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 14ea24f65e6..ffe6f0e3a9f 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -33,6 +33,7 @@ import json import bm_json import tabulate import argparse +import scipy def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -44,15 +45,28 @@ def changed_ratio(n, o): def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 -_INTERESTING = { - 'cpu_time': min_change(10), - 'real_time': min_change(10), - 'locks_per_iteration': min_change(5), - 'allocs_per_iteration': min_change(5), - 'writes_per_iteration': min_change(5), - 'atm_cas_per_iteration': min_change(1), - 'atm_add_per_iteration': min_change(5), -} +_INTERESTING = [ + 'cpu_time', + 'real_time', + 'locks_per_iteration', + 'allocs_per_iteration', + 'writes_per_iteration', + 'atm_cas_per_iteration', + 'atm_add_per_iteration', +] + +_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', + 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', + 'bm_closure', + 'bm_cq', + 'bm_call_create', + 'bm_error', + 'bm_chttp2_hpack', + 'bm_chttp2_transport', + 'bm_pollset', + 'bm_metadata', + 'bm_fullstack_trickle'] argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', @@ -60,59 +74,104 @@ argp.add_argument('-t', '--track', nargs='+', default=sorted(_INTERESTING.keys()), help='Which metrics to track') -argp.add_argument('files', metavar='bm_file.json', type=str, nargs=4, - help='files to diff. ') +argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_error']) +argp.add_argument('-d', '--diff_base', type=str) +argp.add_argument('-r', '--repetitions', type=int, default=5) +argp.add_argument('-p', '--p_threshold', type=float, default=0.05) args = argp.parse_args() -with open(args.files[0]) as f: - js_new_ctr = json.loads(f.read()) -with open(args.files[1]) as f: - js_new_opt = json.loads(f.read()) -with open(args.files[2]) as f: - js_old_ctr = json.loads(f.read()) -with open(args.files[3]) as f: - js_old_opt = json.loads(f.read()) - -new = {} -old = {} - -for row in bm_json.expand_json(js_new_ctr, js_new_opt): - new[row['cpp_name']] = row -for row in bm_json.expand_json(js_old_ctr, js_old_opt): - old[row['cpp_name']] = row - -changed = [] -for fld in args.track: - chk = _INTERESTING[fld] - for bm in new.keys(): - if bm not in old: continue - n = new[bm] - o = old[bm] - if fld not in n or fld not in o: continue - if chk(n[fld], o[fld]): - changed.append((fld, chk)) - break - -headers = ['Benchmark'] + [c[0] for c in changed] + ['Details'] +assert args.diff_base + +def collect1(bm, cfg, ver): + subprocess.check_call(['make', 'clean']) + subprocess.check_call( + ['make', bm_name, + 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) + cmd = ['bins/%s/%s' % (cfg, bm), + '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver), + '--benchmark_out_format=json', + '--benchmark_repetitions=%d' % (args.repetitions) + ] + subprocess.check_call(cmd) + +for bm in args.benchmarks: + collect1(bm, 'opt', 'new') + collect1(bm, 'counters', 'new') + +git_comment = 'Performance differences between this PR and %s\\n' % args.diff_perf + +where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() +subprocess.check_call(['git', 'checkout', args.diff_base]) + +try: + comparables = [] + for bm in args.benchmarks: + try: + collect1(bm, 'opt', 'old') + collect1(bm, 'counters', 'old') + comparables.append(bm_name) + except subprocess.CalledProcessError, e: + pass +finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + + +class Benchmark: + + def __init__(self): + self.samples = { + True: collections.defaultdict(list), + False: collections.defaultdict(list) + } + self.final = {} + + def add_sample(self, data, new): + for f in _INTERESTING: + if f in data: + self.samples[new][f].append(data[f]) + + def process(self): + for f in _INTERESTING: + new = self.samples[True][f] + old = self.samples[False][f] + if not new or not old: continue + p = scipy.stats.ttest_ind(new, old) + if p < args.p_threshold: + self.final[f] = avg(new) - avg(old) + return self.final.keys() + + def row(self, flds): + return [self.final[f] if f in self.final else '' for f in flds] + + +benchmarks = collections.defaultdict(Benchmark) + +for bm in comparables: + with open('%s.counters.new.json' % bm) as f: + js_new_ctr = json.loads(f.read()) + with open('%s.opt.new.json' % bm) as f: + js_new_opt = json.loads(f.read()) + with open('%s.counters.old.json' % bm) as f: + js_old_ctr = json.loads(f.read()) + with open('%s.opt.old.json' % bm) as f: + js_old_opt = json.loads(f.read()) + + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + name = row['cpp_name'] + if name.endswith('_mean') or nme.endswith('_stddev'): continue + benchmarks[name].add_sample(row, True) + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + name = row['cpp_name'] + if name.endswith('_mean') or nme.endswith('_stddev'): continue + benchmarks[name].add_sample(row, False) + +really_interesting = set() +for bm in benchmarks: + really_interesting.update(bm.process()) +fields = [f for f in _INTERESTING if f in really_interesting] + +headers = ['Benchmark'] + fields rows = [] -for bm in sorted(new.keys()): - if bm not in old: continue - row = [bm] - any_changed = False - n = new[bm] - o = old[bm] - details = '' - for fld in args.track: - chk = _INTERESTING[fld] - if fld not in n or fld not in o: continue - if chk(n[fld], o[fld]): - row.append(changed_ratio(n[fld], o[fld])) - if details: details += ', ' - details += '%s:%r-->%r' % (fld, float(o[fld]), float(n[fld])) - any_changed = True - else: - row.append('') - if any_changed: - row.append(details) - rows.append(row) +for name in sorted(benchmarks.keys()): + rows.append([name] + benchmarks[name].row(fields)) print tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py index 57b2636e569..be37139582f 100755 --- a/tools/run_tests/run_microbenchmark.py +++ b/tools/run_tests/run_microbenchmark.py @@ -215,10 +215,6 @@ argp.add_argument('-b', '--benchmarks', nargs='+', type=str, help='Which microbenchmarks should be run') -argp.add_argument('--diff_perf', - default=None, - type=str, - help='Diff microbenchmarks against this git revision') argp.add_argument('--bigquery_upload', default=False, action='store_const', @@ -234,34 +230,6 @@ try: for collect in args.collect: for bm_name in args.benchmarks: collectors[collect](bm_name, args) - if args.diff_perf: - if 'summary' not in args.collect: - for bm_name in args.benchmarks: - run_summary(bm_name, 'opt', bm_name) - run_summary(bm_name, 'counters', bm_name) - where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_perf]) - comparables = [] - subprocess.check_call(['make', 'clean']) - try: - for bm_name in args.benchmarks: - try: - run_summary(bm_name, 'opt', '%s.old' % bm_name) - run_summary(bm_name, 'counters', '%s.old' % bm_name) - comparables.append(bm_name) - except subprocess.CalledProcessError, e: - pass - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - for bm_name in comparables: - diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py', - '%s.counters.json' % bm_name, - '%s.opt.json' % bm_name, - '%s.old.counters.json' % bm_name, - '%s.old.opt.json' % bm_name]).strip() - if diff: - heading('Performance diff: %s' % bm_name) - text(diff) finally: index_html += "\n\n" with open('reports/index.html', 'w') as f: From 00003feddaf531d0a23f230172e74935bcd4f052 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:26:11 -0700 Subject: [PATCH 024/105] Bugfix1 --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index ffe6f0e3a9f..687a04ba7c7 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -70,9 +70,9 @@ _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', - choices=sorted(_INTERESTING.keys()), + choices=sorted(_INTERESTING), nargs='+', - default=sorted(_INTERESTING.keys()), + default=sorted(_INTERESTING), help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_error']) argp.add_argument('-d', '--diff_base', type=str) From ec8dee29e71e0ba119302583441f6770b7880b66 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:26:47 -0700 Subject: [PATCH 025/105] Bugfix2 --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 687a04ba7c7..c10e25087e8 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -34,6 +34,7 @@ import bm_json import tabulate import argparse import scipy +import subprocess def changed_ratio(n, o): if float(o) <= .0001: o = 0 From 039f4dc632d12a0f5fa6aab91f262731fc01e2a5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:27:16 -0700 Subject: [PATCH 026/105] Bugfix3 --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index c10e25087e8..93999af520a 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -86,7 +86,7 @@ assert args.diff_base def collect1(bm, cfg, ver): subprocess.check_call(['make', 'clean']) subprocess.check_call( - ['make', bm_name, + ['make', bm, 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) cmd = ['bins/%s/%s' % (cfg, bm), '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver), From a0af68d1e1490f8646a23bfb240fda45595ac26b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:27:37 -0700 Subject: [PATCH 027/105] Bugfix4 --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 93999af520a..9f85d47ab42 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -35,6 +35,7 @@ import tabulate import argparse import scipy import subprocess +import multiprocessing def changed_ratio(n, o): if float(o) <= .0001: o = 0 From 6e40ea0b652ad76d13f7ac65837ab6612d090af1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:28:22 -0700 Subject: [PATCH 028/105] Bugfix5 --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 9f85d47ab42..a20b46342d4 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -86,6 +86,7 @@ assert args.diff_base def collect1(bm, cfg, ver): subprocess.check_call(['make', 'clean']) + subprocess.check_call(['git', 'submodule', 'update']) subprocess.check_call( ['make', bm, 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) From c523f5df3eb9738db48d766f828e87bee5bb03c4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:37:19 -0700 Subject: [PATCH 029/105] Bugfix6 --- tools/profiling/microbenchmarks/bm_diff.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index a20b46342d4..b018543960b 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -76,7 +76,7 @@ argp.add_argument('-t', '--track', nargs='+', default=sorted(_INTERESTING), help='Which metrics to track') -argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_error']) +argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=5) argp.add_argument('-p', '--p_threshold', type=float, default=0.05) @@ -101,8 +101,6 @@ for bm in args.benchmarks: collect1(bm, 'opt', 'new') collect1(bm, 'counters', 'new') -git_comment = 'Performance differences between this PR and %s\\n' % args.diff_perf - where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) From d9de6b302f7e064bc3bd04b80c19f5e4f5398dc7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:45:10 -0700 Subject: [PATCH 030/105] Bugfix7 --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index b018543960b..f8741521777 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -110,7 +110,7 @@ try: try: collect1(bm, 'opt', 'old') collect1(bm, 'counters', 'old') - comparables.append(bm_name) + comparables.append(bm) except subprocess.CalledProcessError, e: pass finally: From 8aeb8986ea2d0cae2fe9aadc20eea05825f4678f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 21:53:04 -0700 Subject: [PATCH 031/105] Bugfix8 --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index f8741521777..393c8ae239f 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -36,6 +36,7 @@ import argparse import scipy import subprocess import multiprocessing +import collections def changed_ratio(n, o): if float(o) <= .0001: o = 0 From b9d71a2a0eb14c7b40e4e378e1869cf86351be3e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 22:01:10 -0700 Subject: [PATCH 032/105] Bugfix9 --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 393c8ae239f..bc0e8d69fef 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -160,11 +160,11 @@ for bm in comparables: for row in bm_json.expand_json(js_new_ctr, js_new_opt): name = row['cpp_name'] - if name.endswith('_mean') or nme.endswith('_stddev'): continue + if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, True) for row in bm_json.expand_json(js_old_ctr, js_old_opt): name = row['cpp_name'] - if name.endswith('_mean') or nme.endswith('_stddev'): continue + if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, False) really_interesting = set() From f0e7739accd8b24ac6d4fe1ddcd6492db5d26015 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 22:11:57 -0700 Subject: [PATCH 033/105] Bugfix10 --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- tools/profiling/microbenchmarks/bm_json.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index bc0e8d69fef..c725d96626b 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -168,7 +168,7 @@ for bm in comparables: benchmarks[name].add_sample(row, False) really_interesting = set() -for bm in benchmarks: +for bm in benchmarks.values(): really_interesting.update(bm.process()) fields = [f for f in _INTERESTING if f in really_interesting] diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 4695f829f41..78d1016ac01 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -159,6 +159,7 @@ def parse_name(name): def expand_json(js, js2 = None): for bm in js['benchmarks']: + if bm['name'].endswith('_stddev') or bm['name'].endswith('mean'): continue context = js['context'] if 'label' in bm: labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] From 4274925f833722224fddc0372c9a1e5b7ab1f6a8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 22:20:37 -0700 Subject: [PATCH 034/105] Bugfix11 --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index c725d96626b..0d216077943 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -33,7 +33,7 @@ import json import bm_json import tabulate import argparse -import scipy +from scipy import stats import subprocess import multiprocessing import collections @@ -137,7 +137,7 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - p = scipy.stats.ttest_ind(new, old) + p = stats.ttest_ind(new, old) if p < args.p_threshold: self.final[f] = avg(new) - avg(old) return self.final.keys() From cc7d9873f76e3a2d6b799b358a1b4953307927b3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 22:29:15 -0700 Subject: [PATCH 035/105] Bugfix12 --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 0d216077943..7726868a8b8 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -137,6 +137,7 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue + print f, new, old p = stats.ttest_ind(new, old) if p < args.p_threshold: self.final[f] = avg(new) - avg(old) From 4efb5e16dce615b8726cccbbcad4e9d12257b783 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 22:37:00 -0700 Subject: [PATCH 036/105] Bugfix13 --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 7726868a8b8..d81218858e8 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -130,7 +130,7 @@ class Benchmark: def add_sample(self, data, new): for f in _INTERESTING: if f in data: - self.samples[new][f].append(data[f]) + self.samples[new][f].append(float(data[f])) def process(self): for f in _INTERESTING: From a87b138187d5127a4f193072f271ae2121b38a65 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 23:06:10 -0700 Subject: [PATCH 037/105] Bugfix14 --- tools/profiling/microbenchmarks/bm_diff.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index d81218858e8..6b82cbe702d 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -137,12 +137,14 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - print f, new, old - p = stats.ttest_ind(new, old) + p = stats.ttest_ind(new, old)[1] if p < args.p_threshold: self.final[f] = avg(new) - avg(old) return self.final.keys() + def skip(self): + return not self.final + def row(self, flds): return [self.final[f] if f in self.final else '' for f in flds] @@ -176,5 +178,6 @@ fields = [f for f in _INTERESTING if f in really_interesting] headers = ['Benchmark'] + fields rows = [] for name in sorted(benchmarks.keys()): + if benchmarks[name].skip(): continue rows.append([name] + benchmarks[name].row(fields)) print tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') From ed2c292f448f1a414db3f9473171dc88d1802eae Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 27 Mar 2017 23:23:45 -0700 Subject: [PATCH 038/105] Bugfix15 --- tools/profiling/microbenchmarks/bm_diff.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 6b82cbe702d..fe8c76ba215 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -85,6 +85,14 @@ args = argp.parse_args() assert args.diff_base +def avg(lst): + sum = 0 + n = 0 + for el in lst: + sum += el + n += 1 + return sum / n + def collect1(bm, cfg, ver): subprocess.check_call(['make', 'clean']) subprocess.check_call(['git', 'submodule', 'update']) From d3c28275f506b4178ed8ad30e55512528048f090 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 08:27:02 -0700 Subject: [PATCH 039/105] Simpler --- tools/jenkins/run_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index f862bcab308..c2b7af063f8 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -38,6 +38,6 @@ BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_c cd $(dirname $0)/../.. # todo(mattkwong): Change performance test to use microbenchmarking -tools/profiling/microbenchmarks/bm_diff.py -g --diff_base origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN +tools/profiling/microbenchmarks/bm_diff.py -g -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN tools/run_tests/run_performance_tests.py -l c++ node ruby csharp python --netperf --category smoketest From 50b7fc9b241730d238c3143c5d9b98e6ef9afe23 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 08:28:32 -0700 Subject: [PATCH 040/105] More discriminating --- tools/profiling/microbenchmarks/bm_diff.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5c5b6207a48..bc2baaba5ed 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -148,8 +148,9 @@ class Benchmark: old = self.samples[False][f] if not new or not old: continue p = stats.ttest_ind(new, old)[1] - if p < args.p_threshold: - self.final[f] = avg(new) - avg(old) + delta = avg(new) - avg(old) + if p < args.p_threshold and abs(delta) > 0.1: + self.final[f] = delta return self.final.keys() def skip(self): From 409a8da3ed6e63f6cbcd7c37435c448b33e85021 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 08:41:16 -0700 Subject: [PATCH 041/105] syntax --- tools/profiling/microbenchmarks/bm_diff.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index bc2baaba5ed..332f3be4f8a 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -198,6 +198,5 @@ else: print text if args.git_comment: - subprocess.call(['tools/jenkins/comment_on_pr.sh %s' % pipes.quote(text), - stdout=subprocess.PIPE, - shell=True) + subprocess.call(['tools/jenkins/comment_on_pr.sh', text], + stdout=subprocess.PIPE) From 1751acae882b6a6ed31bad01272013f26f0d5f0b Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 28 Mar 2017 02:27:35 +0200 Subject: [PATCH 042/105] Actually setting defaults for grpc_benchmark. --- Makefile | 10 ++++++++++ build.yaml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/Makefile b/Makefile index 62c4f8e9d6b..9be1c18a931 100644 --- a/Makefile +++ b/Makefile @@ -5097,6 +5097,7 @@ PUBLIC_HEADERS_CXX += \ LIBGRPC_BENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_BENCHMARK_SRC)))) +$(LIBGRPC_BENCHMARK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX ifeq ($(NO_SECURE),true) @@ -12838,6 +12839,7 @@ endif endif +$(BM_ARENA_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_arena.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_arena: $(BM_ARENA_OBJS:.o=.dep) @@ -12925,6 +12927,7 @@ endif endif +$(BM_CHTTP2_HPACK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_hpack.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_chttp2_hpack: $(BM_CHTTP2_HPACK_OBJS:.o=.dep) @@ -12968,6 +12971,7 @@ endif endif +$(BM_CHTTP2_TRANSPORT_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_chttp2_transport.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_chttp2_transport: $(BM_CHTTP2_TRANSPORT_OBJS:.o=.dep) @@ -13099,6 +13103,7 @@ endif endif +$(BM_ERROR_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_error.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_error: $(BM_ERROR_OBJS:.o=.dep) @@ -13142,6 +13147,7 @@ endif endif +$(BM_FULLSTACK_STREAMING_PING_PONG_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_fullstack_streaming_ping_pong: $(BM_FULLSTACK_STREAMING_PING_PONG_OBJS:.o=.dep) @@ -13185,6 +13191,7 @@ endif endif +$(BM_FULLSTACK_STREAMING_PUMP_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_fullstack_streaming_pump: $(BM_FULLSTACK_STREAMING_PUMP_OBJS:.o=.dep) @@ -13228,6 +13235,7 @@ endif endif +$(BM_FULLSTACK_TRICKLE_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack_trickle.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_fullstack_trickle: $(BM_FULLSTACK_TRICKLE_OBJS:.o=.dep) @@ -13315,6 +13323,7 @@ endif endif +$(BM_METADATA_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_metadata.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_metadata: $(BM_METADATA_OBJS:.o=.dep) @@ -13358,6 +13367,7 @@ endif endif +$(BM_POLLSET_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX $(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_pollset.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a deps_bm_pollset: $(BM_POLLSET_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index ebaa4c652bb..397cfb8691f 100644 --- a/build.yaml +++ b/build.yaml @@ -1255,6 +1255,7 @@ libs: - grpc++ - grpc_test_util - grpc + defaults: benchmark - name: grpc_cli_libs build: private language: c++ @@ -3089,6 +3090,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3130,6 +3132,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3150,6 +3153,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3212,6 +3216,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3232,6 +3237,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark excluded_poll_engines: - poll - poll-cv @@ -3256,6 +3262,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark excluded_poll_engines: - poll - poll-cv @@ -3280,6 +3287,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark excluded_poll_engines: - poll - poll-cv @@ -3329,6 +3337,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux @@ -3349,6 +3358,7 @@ targets: - gpr args: - --benchmark_min_time=0 + defaults: benchmark platforms: - mac - linux From 134fed3a8543ad59d090c74affe47652dae180b1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 14:27:18 -0700 Subject: [PATCH 043/105] Move src/core/lib/tsi --> src/core/tsi --- BUILD | 16 ++++---- CMakeLists.txt | 12 +++--- Makefile | 18 ++++----- binding.gyp | 6 +-- build.yaml | 18 ++++----- config.m4 | 8 ++-- gRPC-Core.podspec | 26 ++++++------- grpc.gemspec | 16 ++++---- package.xml | 16 ++++---- src/core/{lib => }/tsi/README.md | 0 .../{lib => }/tsi/fake_transport_security.c | 0 .../{lib => }/tsi/fake_transport_security.h | 0 .../{lib => }/tsi/ssl_transport_security.c | 0 .../{lib => }/tsi/ssl_transport_security.h | 0 src/core/{lib => }/tsi/ssl_types.h | 0 src/core/{lib => }/tsi/test_creds/BUILD | 0 src/core/{lib => }/tsi/test_creds/README | 0 .../{lib => }/tsi/test_creds/badclient.key | 0 .../{lib => }/tsi/test_creds/badclient.pem | 0 .../{lib => }/tsi/test_creds/badserver.key | 0 .../{lib => }/tsi/test_creds/badserver.pem | 0 .../{lib => }/tsi/test_creds/ca-openssl.cnf | 0 src/core/{lib => }/tsi/test_creds/ca.key | 0 src/core/{lib => }/tsi/test_creds/ca.pem | 0 src/core/{lib => }/tsi/test_creds/client.key | 0 src/core/{lib => }/tsi/test_creds/client.pem | 0 src/core/{lib => }/tsi/test_creds/server0.key | 0 src/core/{lib => }/tsi/test_creds/server0.pem | 0 .../tsi/test_creds/server1-openssl.cnf | 0 src/core/{lib => }/tsi/test_creds/server1.key | 0 src/core/{lib => }/tsi/test_creds/server1.pem | 0 src/core/{lib => }/tsi/transport_security.c | 0 src/core/{lib => }/tsi/transport_security.h | 0 .../tsi/transport_security_interface.h | 0 src/python/grpcio/grpc_core_dependencies.py | 6 +-- tools/doxygen/Doxyfile.core.internal | 18 ++++----- .../generated/sources_and_headers.json | 26 ++++++------- vsprojects/vcxproj/grpc/grpc.vcxproj | 16 ++++---- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 38 +++++++++---------- 39 files changed, 120 insertions(+), 120 deletions(-) rename src/core/{lib => }/tsi/README.md (100%) rename src/core/{lib => }/tsi/fake_transport_security.c (100%) rename src/core/{lib => }/tsi/fake_transport_security.h (100%) rename src/core/{lib => }/tsi/ssl_transport_security.c (100%) rename src/core/{lib => }/tsi/ssl_transport_security.h (100%) rename src/core/{lib => }/tsi/ssl_types.h (100%) rename src/core/{lib => }/tsi/test_creds/BUILD (100%) rename src/core/{lib => }/tsi/test_creds/README (100%) rename src/core/{lib => }/tsi/test_creds/badclient.key (100%) rename src/core/{lib => }/tsi/test_creds/badclient.pem (100%) rename src/core/{lib => }/tsi/test_creds/badserver.key (100%) rename src/core/{lib => }/tsi/test_creds/badserver.pem (100%) rename src/core/{lib => }/tsi/test_creds/ca-openssl.cnf (100%) rename src/core/{lib => }/tsi/test_creds/ca.key (100%) rename src/core/{lib => }/tsi/test_creds/ca.pem (100%) rename src/core/{lib => }/tsi/test_creds/client.key (100%) rename src/core/{lib => }/tsi/test_creds/client.pem (100%) rename src/core/{lib => }/tsi/test_creds/server0.key (100%) rename src/core/{lib => }/tsi/test_creds/server0.pem (100%) rename src/core/{lib => }/tsi/test_creds/server1-openssl.cnf (100%) rename src/core/{lib => }/tsi/test_creds/server1.key (100%) rename src/core/{lib => }/tsi/test_creds/server1.pem (100%) rename src/core/{lib => }/tsi/transport_security.c (100%) rename src/core/{lib => }/tsi/transport_security.h (100%) rename src/core/{lib => }/tsi/transport_security_interface.h (100%) diff --git a/BUILD b/BUILD index d6598487fba..f6187e0998e 100644 --- a/BUILD +++ b/BUILD @@ -1128,16 +1128,16 @@ grpc_cc_library( grpc_cc_library( name = "tsi", srcs = [ - "src/core/lib/tsi/fake_transport_security.c", - "src/core/lib/tsi/ssl_transport_security.c", - "src/core/lib/tsi/transport_security.c", + "src/core/tsi/fake_transport_security.c", + "src/core/tsi/ssl_transport_security.c", + "src/core/tsi/transport_security.c", ], hdrs = [ - "src/core/lib/tsi/fake_transport_security.h", - "src/core/lib/tsi/ssl_transport_security.h", - "src/core/lib/tsi/ssl_types.h", - "src/core/lib/tsi/transport_security.h", - "src/core/lib/tsi/transport_security_interface.h", + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h", ], external_deps = [ "libssl", diff --git a/CMakeLists.txt b/CMakeLists.txt index f54e78b560d..7b8e49e1262 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1078,9 +1078,9 @@ add_library(grpc src/core/lib/security/transport/tsi_error.c src/core/lib/security/util/json_util.c src/core/lib/surface/init_secure.c - src/core/lib/tsi/fake_transport_security.c - src/core/lib/tsi/ssl_transport_security.c - src/core/lib/tsi/transport_security.c + src/core/tsi/fake_transport_security.c + src/core/tsi/ssl_transport_security.c + src/core/tsi/transport_security.c src/core/ext/transport/chttp2/server/chttp2_server.c src/core/ext/transport/chttp2/client/secure/secure_channel_create.c src/core/ext/client_channel/channel_connectivity.c @@ -1422,9 +1422,9 @@ add_library(grpc_cronet src/core/lib/security/transport/tsi_error.c src/core/lib/security/util/json_util.c src/core/lib/surface/init_secure.c - src/core/lib/tsi/fake_transport_security.c - src/core/lib/tsi/ssl_transport_security.c - src/core/lib/tsi/transport_security.c + src/core/tsi/fake_transport_security.c + src/core/tsi/ssl_transport_security.c + src/core/tsi/transport_security.c src/core/ext/transport/chttp2/client/chttp2_connector.c src/core/ext/load_reporting/load_reporting.c src/core/ext/load_reporting/load_reporting_filter.c diff --git a/Makefile b/Makefile index bfc43aab28e..425160403fe 100644 --- a/Makefile +++ b/Makefile @@ -2972,9 +2972,9 @@ LIBGRPC_SRC = \ src/core/lib/security/transport/tsi_error.c \ src/core/lib/security/util/json_util.c \ src/core/lib/surface/init_secure.c \ - src/core/lib/tsi/fake_transport_security.c \ - src/core/lib/tsi/ssl_transport_security.c \ - src/core/lib/tsi/transport_security.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \ src/core/ext/client_channel/channel_connectivity.c \ @@ -3314,9 +3314,9 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/security/transport/tsi_error.c \ src/core/lib/security/util/json_util.c \ src/core/lib/surface/init_secure.c \ - src/core/lib/tsi/fake_transport_security.c \ - src/core/lib/tsi/ssl_transport_security.c \ - src/core/lib/tsi/transport_security.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ src/core/ext/transport/chttp2/client/chttp2_connector.c \ src/core/ext/load_reporting/load_reporting.c \ src/core/ext/load_reporting/load_reporting_filter.c \ @@ -18807,11 +18807,11 @@ src/core/lib/security/transport/server_auth_filter.c: $(OPENSSL_DEP) src/core/lib/security/transport/tsi_error.c: $(OPENSSL_DEP) src/core/lib/security/util/json_util.c: $(OPENSSL_DEP) src/core/lib/surface/init_secure.c: $(OPENSSL_DEP) -src/core/lib/tsi/fake_transport_security.c: $(OPENSSL_DEP) -src/core/lib/tsi/ssl_transport_security.c: $(OPENSSL_DEP) -src/core/lib/tsi/transport_security.c: $(OPENSSL_DEP) src/core/plugin_registry/grpc_cronet_plugin_registry.c: $(OPENSSL_DEP) src/core/plugin_registry/grpc_plugin_registry.c: $(OPENSSL_DEP) +src/core/tsi/fake_transport_security.c: $(OPENSSL_DEP) +src/core/tsi/ssl_transport_security.c: $(OPENSSL_DEP) +src/core/tsi/transport_security.c: $(OPENSSL_DEP) src/cpp/client/cronet_credentials.cc: $(OPENSSL_DEP) src/cpp/client/secure_credentials.cc: $(OPENSSL_DEP) src/cpp/common/auth_property_iterator.cc: $(OPENSSL_DEP) diff --git a/binding.gyp b/binding.gyp index b91f5f61e21..f1cef607ec0 100644 --- a/binding.gyp +++ b/binding.gyp @@ -785,9 +785,9 @@ 'src/core/lib/security/transport/tsi_error.c', 'src/core/lib/security/util/json_util.c', 'src/core/lib/surface/init_secure.c', - 'src/core/lib/tsi/fake_transport_security.c', - 'src/core/lib/tsi/ssl_transport_security.c', - 'src/core/lib/tsi/transport_security.c', + 'src/core/tsi/fake_transport_security.c', + 'src/core/tsi/ssl_transport_security.c', + 'src/core/tsi/transport_security.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', 'src/core/ext/client_channel/channel_connectivity.c', diff --git a/build.yaml b/build.yaml index 6e590f4ca6b..6a89b71fb2f 100644 --- a/build.yaml +++ b/build.yaml @@ -770,15 +770,15 @@ filegroups: - third_party/nanopb/pb_encode.c - name: tsi headers: - - src/core/lib/tsi/fake_transport_security.h - - src/core/lib/tsi/ssl_transport_security.h - - src/core/lib/tsi/ssl_types.h - - src/core/lib/tsi/transport_security.h - - src/core/lib/tsi/transport_security_interface.h - src: - - src/core/lib/tsi/fake_transport_security.c - - src/core/lib/tsi/ssl_transport_security.c - - src/core/lib/tsi/transport_security.c + - src/core/tsi/fake_transport_security.h + - src/core/tsi/ssl_transport_security.h + - src/core/tsi/ssl_types.h + - src/core/tsi/transport_security.h + - src/core/tsi/transport_security_interface.h + src: + - src/core/tsi/fake_transport_security.c + - src/core/tsi/ssl_transport_security.c + - src/core/tsi/transport_security.c deps: - gpr secure: true diff --git a/config.m4 b/config.m4 index b6f0bff7ef0..6ce7d70c5b6 100644 --- a/config.m4 +++ b/config.m4 @@ -253,9 +253,9 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/security/transport/tsi_error.c \ src/core/lib/security/util/json_util.c \ src/core/lib/surface/init_secure.c \ - src/core/lib/tsi/fake_transport_security.c \ - src/core/lib/tsi/ssl_transport_security.c \ - src/core/lib/tsi/transport_security.c \ + src/core/tsi/fake_transport_security.c \ + src/core/tsi/ssl_transport_security.c \ + src/core/tsi/transport_security.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \ src/core/ext/client_channel/channel_connectivity.c \ @@ -668,8 +668,8 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/support) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/surface) PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/transport) - PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/tsi) PHP_ADD_BUILD_DIR($ext_builddir/src/core/plugin_registry) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi) PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto) PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/aes) PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/asn1) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 8ea9e5fbdec..97bd3c28643 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -404,11 +404,11 @@ Pod::Spec.new do |s| 'src/core/lib/security/transport/tsi_error.h', 'src/core/lib/security/util/b64.h', 'src/core/lib/security/util/json_util.h', - 'src/core/lib/tsi/fake_transport_security.h', - 'src/core/lib/tsi/ssl_transport_security.h', - 'src/core/lib/tsi/ssl_types.h', - 'src/core/lib/tsi/transport_security.h', - 'src/core/lib/tsi/transport_security_interface.h', + 'src/core/tsi/fake_transport_security.h', + 'src/core/tsi/ssl_transport_security.h', + 'src/core/tsi/ssl_types.h', + 'src/core/tsi/transport_security.h', + 'src/core/tsi/transport_security_interface.h', 'src/core/ext/transport/chttp2/server/chttp2_server.h', 'src/core/ext/client_channel/client_channel.h', 'src/core/ext/client_channel/client_channel_factory.h', @@ -628,9 +628,9 @@ Pod::Spec.new do |s| 'src/core/lib/security/transport/tsi_error.c', 'src/core/lib/security/util/json_util.c', 'src/core/lib/surface/init_secure.c', - 'src/core/lib/tsi/fake_transport_security.c', - 'src/core/lib/tsi/ssl_transport_security.c', - 'src/core/lib/tsi/transport_security.c', + 'src/core/tsi/fake_transport_security.c', + 'src/core/tsi/ssl_transport_security.c', + 'src/core/tsi/transport_security.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', 'src/core/ext/client_channel/channel_connectivity.c', @@ -852,11 +852,11 @@ Pod::Spec.new do |s| 'src/core/lib/security/transport/tsi_error.h', 'src/core/lib/security/util/b64.h', 'src/core/lib/security/util/json_util.h', - 'src/core/lib/tsi/fake_transport_security.h', - 'src/core/lib/tsi/ssl_transport_security.h', - 'src/core/lib/tsi/ssl_types.h', - 'src/core/lib/tsi/transport_security.h', - 'src/core/lib/tsi/transport_security_interface.h', + 'src/core/tsi/fake_transport_security.h', + 'src/core/tsi/ssl_transport_security.h', + 'src/core/tsi/ssl_types.h', + 'src/core/tsi/transport_security.h', + 'src/core/tsi/transport_security_interface.h', 'src/core/ext/transport/chttp2/server/chttp2_server.h', 'src/core/ext/client_channel/client_channel.h', 'src/core/ext/client_channel/client_channel_factory.h', diff --git a/grpc.gemspec b/grpc.gemspec index 06da332bed5..cb51da1b574 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -320,11 +320,11 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/transport/tsi_error.h ) s.files += %w( src/core/lib/security/util/b64.h ) s.files += %w( src/core/lib/security/util/json_util.h ) - s.files += %w( src/core/lib/tsi/fake_transport_security.h ) - s.files += %w( src/core/lib/tsi/ssl_transport_security.h ) - s.files += %w( src/core/lib/tsi/ssl_types.h ) - s.files += %w( src/core/lib/tsi/transport_security.h ) - s.files += %w( src/core/lib/tsi/transport_security_interface.h ) + s.files += %w( src/core/tsi/fake_transport_security.h ) + s.files += %w( src/core/tsi/ssl_transport_security.h ) + s.files += %w( src/core/tsi/ssl_types.h ) + s.files += %w( src/core/tsi/transport_security.h ) + s.files += %w( src/core/tsi/transport_security_interface.h ) s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.h ) s.files += %w( src/core/ext/client_channel/client_channel.h ) s.files += %w( src/core/ext/client_channel/client_channel_factory.h ) @@ -544,9 +544,9 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/transport/tsi_error.c ) s.files += %w( src/core/lib/security/util/json_util.c ) s.files += %w( src/core/lib/surface/init_secure.c ) - s.files += %w( src/core/lib/tsi/fake_transport_security.c ) - s.files += %w( src/core/lib/tsi/ssl_transport_security.c ) - s.files += %w( src/core/lib/tsi/transport_security.c ) + s.files += %w( src/core/tsi/fake_transport_security.c ) + s.files += %w( src/core/tsi/ssl_transport_security.c ) + s.files += %w( src/core/tsi/transport_security.c ) s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.c ) s.files += %w( src/core/ext/transport/chttp2/client/secure/secure_channel_create.c ) s.files += %w( src/core/ext/client_channel/channel_connectivity.c ) diff --git a/package.xml b/package.xml index 5d585845556..afac1800bed 100644 --- a/package.xml +++ b/package.xml @@ -329,11 +329,11 @@ - - - - - + + + + + @@ -553,9 +553,9 @@ - - - + + + diff --git a/src/core/lib/tsi/README.md b/src/core/tsi/README.md similarity index 100% rename from src/core/lib/tsi/README.md rename to src/core/tsi/README.md diff --git a/src/core/lib/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c similarity index 100% rename from src/core/lib/tsi/fake_transport_security.c rename to src/core/tsi/fake_transport_security.c diff --git a/src/core/lib/tsi/fake_transport_security.h b/src/core/tsi/fake_transport_security.h similarity index 100% rename from src/core/lib/tsi/fake_transport_security.h rename to src/core/tsi/fake_transport_security.h diff --git a/src/core/lib/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c similarity index 100% rename from src/core/lib/tsi/ssl_transport_security.c rename to src/core/tsi/ssl_transport_security.c diff --git a/src/core/lib/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h similarity index 100% rename from src/core/lib/tsi/ssl_transport_security.h rename to src/core/tsi/ssl_transport_security.h diff --git a/src/core/lib/tsi/ssl_types.h b/src/core/tsi/ssl_types.h similarity index 100% rename from src/core/lib/tsi/ssl_types.h rename to src/core/tsi/ssl_types.h diff --git a/src/core/lib/tsi/test_creds/BUILD b/src/core/tsi/test_creds/BUILD similarity index 100% rename from src/core/lib/tsi/test_creds/BUILD rename to src/core/tsi/test_creds/BUILD diff --git a/src/core/lib/tsi/test_creds/README b/src/core/tsi/test_creds/README similarity index 100% rename from src/core/lib/tsi/test_creds/README rename to src/core/tsi/test_creds/README diff --git a/src/core/lib/tsi/test_creds/badclient.key b/src/core/tsi/test_creds/badclient.key similarity index 100% rename from src/core/lib/tsi/test_creds/badclient.key rename to src/core/tsi/test_creds/badclient.key diff --git a/src/core/lib/tsi/test_creds/badclient.pem b/src/core/tsi/test_creds/badclient.pem similarity index 100% rename from src/core/lib/tsi/test_creds/badclient.pem rename to src/core/tsi/test_creds/badclient.pem diff --git a/src/core/lib/tsi/test_creds/badserver.key b/src/core/tsi/test_creds/badserver.key similarity index 100% rename from src/core/lib/tsi/test_creds/badserver.key rename to src/core/tsi/test_creds/badserver.key diff --git a/src/core/lib/tsi/test_creds/badserver.pem b/src/core/tsi/test_creds/badserver.pem similarity index 100% rename from src/core/lib/tsi/test_creds/badserver.pem rename to src/core/tsi/test_creds/badserver.pem diff --git a/src/core/lib/tsi/test_creds/ca-openssl.cnf b/src/core/tsi/test_creds/ca-openssl.cnf similarity index 100% rename from src/core/lib/tsi/test_creds/ca-openssl.cnf rename to src/core/tsi/test_creds/ca-openssl.cnf diff --git a/src/core/lib/tsi/test_creds/ca.key b/src/core/tsi/test_creds/ca.key similarity index 100% rename from src/core/lib/tsi/test_creds/ca.key rename to src/core/tsi/test_creds/ca.key diff --git a/src/core/lib/tsi/test_creds/ca.pem b/src/core/tsi/test_creds/ca.pem similarity index 100% rename from src/core/lib/tsi/test_creds/ca.pem rename to src/core/tsi/test_creds/ca.pem diff --git a/src/core/lib/tsi/test_creds/client.key b/src/core/tsi/test_creds/client.key similarity index 100% rename from src/core/lib/tsi/test_creds/client.key rename to src/core/tsi/test_creds/client.key diff --git a/src/core/lib/tsi/test_creds/client.pem b/src/core/tsi/test_creds/client.pem similarity index 100% rename from src/core/lib/tsi/test_creds/client.pem rename to src/core/tsi/test_creds/client.pem diff --git a/src/core/lib/tsi/test_creds/server0.key b/src/core/tsi/test_creds/server0.key similarity index 100% rename from src/core/lib/tsi/test_creds/server0.key rename to src/core/tsi/test_creds/server0.key diff --git a/src/core/lib/tsi/test_creds/server0.pem b/src/core/tsi/test_creds/server0.pem similarity index 100% rename from src/core/lib/tsi/test_creds/server0.pem rename to src/core/tsi/test_creds/server0.pem diff --git a/src/core/lib/tsi/test_creds/server1-openssl.cnf b/src/core/tsi/test_creds/server1-openssl.cnf similarity index 100% rename from src/core/lib/tsi/test_creds/server1-openssl.cnf rename to src/core/tsi/test_creds/server1-openssl.cnf diff --git a/src/core/lib/tsi/test_creds/server1.key b/src/core/tsi/test_creds/server1.key similarity index 100% rename from src/core/lib/tsi/test_creds/server1.key rename to src/core/tsi/test_creds/server1.key diff --git a/src/core/lib/tsi/test_creds/server1.pem b/src/core/tsi/test_creds/server1.pem similarity index 100% rename from src/core/lib/tsi/test_creds/server1.pem rename to src/core/tsi/test_creds/server1.pem diff --git a/src/core/lib/tsi/transport_security.c b/src/core/tsi/transport_security.c similarity index 100% rename from src/core/lib/tsi/transport_security.c rename to src/core/tsi/transport_security.c diff --git a/src/core/lib/tsi/transport_security.h b/src/core/tsi/transport_security.h similarity index 100% rename from src/core/lib/tsi/transport_security.h rename to src/core/tsi/transport_security.h diff --git a/src/core/lib/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h similarity index 100% rename from src/core/lib/tsi/transport_security_interface.h rename to src/core/tsi/transport_security_interface.h diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 258c65c08e9..970c70bb1b0 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -247,9 +247,9 @@ CORE_SOURCE_FILES = [ 'src/core/lib/security/transport/tsi_error.c', 'src/core/lib/security/util/json_util.c', 'src/core/lib/surface/init_secure.c', - 'src/core/lib/tsi/fake_transport_security.c', - 'src/core/lib/tsi/ssl_transport_security.c', - 'src/core/lib/tsi/transport_security.c', + 'src/core/tsi/fake_transport_security.c', + 'src/core/tsi/ssl_transport_security.c', + 'src/core/tsi/transport_security.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', 'src/core/ext/client_channel/channel_connectivity.c', diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index d09ddaa04dc..a15b624f34a 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1359,16 +1359,16 @@ src/core/lib/transport/transport.c \ src/core/lib/transport/transport.h \ src/core/lib/transport/transport_impl.h \ src/core/lib/transport/transport_op_string.c \ -src/core/lib/tsi/README.md \ -src/core/lib/tsi/fake_transport_security.c \ -src/core/lib/tsi/fake_transport_security.h \ -src/core/lib/tsi/ssl_transport_security.c \ -src/core/lib/tsi/ssl_transport_security.h \ -src/core/lib/tsi/ssl_types.h \ -src/core/lib/tsi/transport_security.c \ -src/core/lib/tsi/transport_security.h \ -src/core/lib/tsi/transport_security_interface.h \ src/core/plugin_registry/grpc_plugin_registry.c \ +src/core/tsi/README.md \ +src/core/tsi/fake_transport_security.c \ +src/core/tsi/fake_transport_security.h \ +src/core/tsi/ssl_transport_security.c \ +src/core/tsi/ssl_transport_security.h \ +src/core/tsi/ssl_types.h \ +src/core/tsi/transport_security.c \ +src/core/tsi/transport_security.h \ +src/core/tsi/transport_security_interface.h \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_common.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fd12ea5c471..446a27e6cfd 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8517,24 +8517,24 @@ "gpr" ], "headers": [ - "src/core/lib/tsi/fake_transport_security.h", - "src/core/lib/tsi/ssl_transport_security.h", - "src/core/lib/tsi/ssl_types.h", - "src/core/lib/tsi/transport_security.h", - "src/core/lib/tsi/transport_security_interface.h" + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h" ], "is_filegroup": true, "language": "c", "name": "tsi", "src": [ - "src/core/lib/tsi/fake_transport_security.c", - "src/core/lib/tsi/fake_transport_security.h", - "src/core/lib/tsi/ssl_transport_security.c", - "src/core/lib/tsi/ssl_transport_security.h", - "src/core/lib/tsi/ssl_types.h", - "src/core/lib/tsi/transport_security.c", - "src/core/lib/tsi/transport_security.h", - "src/core/lib/tsi/transport_security_interface.h" + "src/core/tsi/fake_transport_security.c", + "src/core/tsi/fake_transport_security.h", + "src/core/tsi/ssl_transport_security.c", + "src/core/tsi/ssl_transport_security.h", + "src/core/tsi/ssl_types.h", + "src/core/tsi/transport_security.c", + "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_interface.h" ], "third_party": false, "type": "filegroup" diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 7abff218dde..22e61e768a2 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -448,11 +448,11 @@ - - - - - + + + + + @@ -845,11 +845,11 @@ - + - + - + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index aa3e5a77aa6..95e96142a92 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -514,14 +514,14 @@ src\core\lib\surface - - src\core\lib\tsi + + src\core\tsi - - src\core\lib\tsi + + src\core\tsi - - src\core\lib\tsi + + src\core\tsi src\core\ext\transport\chttp2\server @@ -1238,20 +1238,20 @@ src\core\lib\security\util - - src\core\lib\tsi + + src\core\tsi - - src\core\lib\tsi + + src\core\tsi - - src\core\lib\tsi + + src\core\tsi - - src\core\lib\tsi + + src\core\tsi - - src\core\lib\tsi + + src\core\tsi src\core\ext\transport\chttp2\server @@ -1571,12 +1571,12 @@ {e9d0d3fc-c100-f3e6-89b8-649f241155bf} - - {95ad2811-c8d0-7a42-2a73-baf03fcbf699} - {02bec99b-ff39-88d7-9dea-e0ff9f4a2701} + + {0b0f9ab1-efa4-7f03-e446-6fb9b5227e84} + {aaab30a4-2a15-732e-c141-3fbc0f0f5a7a} From e2e34b8812f36eafa4fb2391bd3eb72d17d9789d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 14:36:50 -0700 Subject: [PATCH 044/105] Line counter --- tools/line_count/summarize-history.py | 17 +++++++++++++++++ tools/line_count/yaml2csv.py | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100755 tools/line_count/summarize-history.py create mode 100755 tools/line_count/yaml2csv.py diff --git a/tools/line_count/summarize-history.py b/tools/line_count/summarize-history.py new file mode 100755 index 00000000000..edaaf6517bb --- /dev/null +++ b/tools/line_count/summarize-history.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import subprocess +import datetime + +def daterange(start, end): + for n in range(int((end - start).days)): + yield start + datetime.timedelta(n) + +start_date = datetime.date(2014, 11, 26) +end_date = datetime.date(2017, 3, 26) + +for dt in daterange(start_date, end_date): + dmy = dt.strftime('%Y-%m-%d') + print dmy + subprocess.check_call(['tools/line_count/yaml2csv.py', '-i', '../count/%s.yaml' % dmy, '-d', dmy, '-o', '../count/%s.csv' % dmy]) + diff --git a/tools/line_count/yaml2csv.py b/tools/line_count/yaml2csv.py new file mode 100755 index 00000000000..159901ac412 --- /dev/null +++ b/tools/line_count/yaml2csv.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import yaml +import argparse +import datetime +import csv + +argp = argparse.ArgumentParser(description='Convert cloc yaml to bigquery csv') +argp.add_argument('-i', '--input', type=str) +argp.add_argument('-d', '--date', type=str, default=datetime.date.today().strftime('%Y-%m-%d')) +argp.add_argument('-o', '--output', type=str, default='out.csv') +args = argp.parse_args() + +data = yaml.load(open(args.input).read()) +with open(args.output, 'w') as outf: + writer = csv.DictWriter(outf, ['date', 'name', 'language', 'code', 'comment', 'blank']) + for key, value in data.iteritems(): + if key == 'header': continue + if key == 'SUM': continue + if key.startswith('third_party/'): continue + row = {'name': key, 'date': args.date} + row.update(value) + writer.writerow(row) + From 0fc13f9d807f6db35ace60c8f6ec6ad093765bca Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 14:38:10 -0700 Subject: [PATCH 045/105] Backfill --- tools/line_count/collect-history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/line_count/collect-history.py b/tools/line_count/collect-history.py index 0de95f2bac7..e5e8b42fdc2 100755 --- a/tools/line_count/collect-history.py +++ b/tools/line_count/collect-history.py @@ -7,8 +7,8 @@ def daterange(start, end): for n in range(int((end - start).days)): yield start + datetime.timedelta(n) -start_date = datetime.date(2014, 11, 21) -end_date = datetime.date(2017, 3, 26) +start_date = datetime.date(2017, 3, 26) +end_date = datetime.date(2017, 3, 29) for dt in daterange(start_date, end_date): dmy = dt.strftime('%Y-%m-%d') From 01704b41b1d5d50360d9e92d46046e49206dbfe5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 14:43:41 -0700 Subject: [PATCH 046/105] Backfill --- tools/line_count/summarize-history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/line_count/summarize-history.py b/tools/line_count/summarize-history.py index edaaf6517bb..af87ddb31a9 100755 --- a/tools/line_count/summarize-history.py +++ b/tools/line_count/summarize-history.py @@ -7,8 +7,8 @@ def daterange(start, end): for n in range(int((end - start).days)): yield start + datetime.timedelta(n) -start_date = datetime.date(2014, 11, 26) -end_date = datetime.date(2017, 3, 26) +start_date = datetime.date(2017, 3, 26) +end_date = datetime.date(2017, 3, 29) for dt in daterange(start_date, end_date): dmy = dt.strftime('%Y-%m-%d') From 45f96aa3a7bcc36626c6cea7be12c9348aad42f4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 14:44:51 -0700 Subject: [PATCH 047/105] Fix copyright --- tools/line_count/collect-history.py | 28 ++++++++++++++++++++++++++ tools/line_count/summarize-history.py | 29 +++++++++++++++++++++++++++ tools/line_count/yaml2csv.py | 29 +++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/tools/line_count/collect-history.py b/tools/line_count/collect-history.py index e5e8b42fdc2..858a6e1d30c 100755 --- a/tools/line_count/collect-history.py +++ b/tools/line_count/collect-history.py @@ -1,4 +1,32 @@ #!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import subprocess import datetime diff --git a/tools/line_count/summarize-history.py b/tools/line_count/summarize-history.py index af87ddb31a9..756dc79a4dc 100755 --- a/tools/line_count/summarize-history.py +++ b/tools/line_count/summarize-history.py @@ -1,4 +1,33 @@ #!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import subprocess import datetime diff --git a/tools/line_count/yaml2csv.py b/tools/line_count/yaml2csv.py index 159901ac412..9bda09fd087 100755 --- a/tools/line_count/yaml2csv.py +++ b/tools/line_count/yaml2csv.py @@ -1,4 +1,33 @@ #!/usr/bin/env python +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import yaml import argparse From 14d59ac25f29c7f0c4ca2e9cf9bea2deb9dcf6a0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 14:48:37 -0700 Subject: [PATCH 048/105] Add a collector script --- tools/line_count/collect-now.sh | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tools/line_count/collect-now.sh diff --git a/tools/line_count/collect-now.sh b/tools/line_count/collect-now.sh new file mode 100644 index 00000000000..2db26298dc4 --- /dev/null +++ b/tools/line_count/collect-now.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cloc --vcs=git --by-file --yaml --out=cloc.yaml . +tools/line_count/yaml2csv -i cloc.yaml -d `date +%Y-%m-%d` -o cloc.csv +bq load line_counts.grpc cloc.csv + From b29f1fe3a2765e401a34dfa731aa150a6c65ed53 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 15:49:23 -0700 Subject: [PATCH 049/105] Update include paths --- doc/interop-test-descriptions.md | 4 ++-- src/core/lib/http/httpcli_security_connector.c | 2 +- src/core/lib/security/credentials/jwt/jwt_verifier.c | 2 +- src/core/lib/security/transport/secure_endpoint.c | 2 +- src/core/lib/security/transport/security_connector.c | 4 ++-- src/core/lib/security/transport/security_connector.h | 2 +- src/core/lib/security/transport/tsi_error.h | 2 +- src/core/lib/surface/init_secure.c | 2 +- src/core/tsi/fake_transport_security.c | 4 ++-- src/core/tsi/fake_transport_security.h | 2 +- src/core/tsi/ssl_transport_security.c | 6 +++--- src/core/tsi/ssl_transport_security.h | 2 +- src/core/tsi/transport_security.c | 2 +- src/core/tsi/transport_security.h | 2 +- test/core/bad_ssl/servers/cert.c | 4 ++-- test/core/handshake/BUILD | 12 ++++++------ test/core/handshake/client_ssl.c | 10 +++++----- test/core/handshake/server_ssl.c | 8 ++++---- test/core/http/test_server.py | 4 ++-- test/core/security/secure_endpoint_test.c | 2 +- test/core/security/security_connector_test.c | 4 ++-- test/core/surface/server_chttp2_test.c | 2 +- test/core/tsi/transport_security_test.c | 6 +++--- tools/http2_interop/http2interop_test.go | 2 +- tools/run_tests/run_tests.py | 2 +- 25 files changed, 47 insertions(+), 47 deletions(-) diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index da3b9767443..66a034d6307 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -27,7 +27,7 @@ Clients should accept these arguments: * Whether to use a plaintext or encrypted connection * --use_test_ca=BOOLEAN * Whether to replace platform root CAs with - [ca.pem](https://github.com/grpc/grpc/blob/master/src/core/lib/tsi/test_creds/ca.pem) + [ca.pem](https://github.com/grpc/grpc/blob/master/src/core/tsi/test_creds/ca.pem) as the CA root * --default_service_account=ACCOUNT_EMAIL * Email of the GCE default service account. @@ -969,7 +969,7 @@ Servers should accept these arguments: * Whether to use a plaintext or encrypted connection Servers must support TLS with ALPN. They should use -[server1.pem](https://github.com/grpc/grpc/blob/master/src/core/lib/tsi/test_creds/server1.pem) +[server1.pem](https://github.com/grpc/grpc/blob/master/src/core/tsi/test_creds/server1.pem) for their certificate. ### EmptyCall diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index be6a6d618a1..fc338342e48 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -43,7 +43,7 @@ #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" -#include "src/core/lib/tsi/ssl_transport_security.h" +#include "src/core/tsi/ssl_transport_security.h" typedef struct { grpc_channel_security_connector base; diff --git a/src/core/lib/security/credentials/jwt/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c index 5c59cf0f4ad..b10a5da2a20 100644 --- a/src/core/lib/security/credentials/jwt/jwt_verifier.c +++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c @@ -48,7 +48,7 @@ #include "src/core/lib/security/util/b64.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" -#include "src/core/lib/tsi/ssl_types.h" +#include "src/core/tsi/ssl_types.h" /* --- Utils. --- */ diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 568d70fa38a..24da949e486 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -49,7 +49,7 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" #define STAGING_BUFFER_SIZE 8192 diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index b0cbc83639c..2b517061611 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -54,8 +54,8 @@ #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" -#include "src/core/lib/tsi/fake_transport_security.h" -#include "src/core/lib/tsi/ssl_transport_security.h" +#include "src/core/tsi/fake_transport_security.h" +#include "src/core/tsi/ssl_transport_security.h" /* -- Constants. -- */ diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index 3df2fecd393..cf56cb31830 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -39,7 +39,7 @@ #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/tcp_server.h" -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" /* --- status enum. --- */ diff --git a/src/core/lib/security/transport/tsi_error.h b/src/core/lib/security/transport/tsi_error.h index 636fbb89cf5..b84693b5de2 100644 --- a/src/core/lib/security/transport/tsi_error.h +++ b/src/core/lib/security/transport/tsi_error.h @@ -35,7 +35,7 @@ #define GRPC_CORE_LIB_SECURITY_TRANSPORT_TSI_ERROR_H #include "src/core/lib/iomgr/error.h" -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" grpc_error *grpc_set_tsi_error_result(grpc_error *error, tsi_result result); diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c index 46b9a8f922c..921ef87e366 100644 --- a/src/core/lib/surface/init_secure.c +++ b/src/core/lib/surface/init_secure.c @@ -43,7 +43,7 @@ #include "src/core/lib/security/transport/security_connector.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/surface/channel_init.h" -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" void grpc_security_pre_init(void) { grpc_register_tracer("secure_endpoint", &grpc_trace_secure_endpoint); diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index bbe323df3bb..822fad51cb3 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/tsi/fake_transport_security.h" +#include "src/core/tsi/fake_transport_security.h" #include #include @@ -40,7 +40,7 @@ #include #include #include -#include "src/core/lib/tsi/transport_security.h" +#include "src/core/tsi/transport_security.h" /* --- Constants. ---*/ #define TSI_FAKE_FRAME_HEADER_SIZE 4 diff --git a/src/core/tsi/fake_transport_security.h b/src/core/tsi/fake_transport_security.h index 54a9469b584..7c1c8ed195c 100644 --- a/src/core/tsi/fake_transport_security.h +++ b/src/core/tsi/fake_transport_security.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H #define GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" #ifdef __cplusplus extern "C" { diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 53aabdb9267..a0325cc1835 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/tsi/ssl_transport_security.h" +#include "src/core/tsi/ssl_transport_security.h" #include @@ -60,8 +60,8 @@ #include #include -#include "src/core/lib/tsi/ssl_types.h" -#include "src/core/lib/tsi/transport_security.h" +#include "src/core/tsi/ssl_types.h" +#include "src/core/tsi/transport_security.h" /* --- Constants. ---*/ diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 7407246118a..afc25da98d2 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H #define GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" #ifdef __cplusplus extern "C" { diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index 2cbf381c88d..a3e42e87ec8 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/tsi/transport_security.h" +#include "src/core/tsi/transport_security.h" #include #include diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index aaf110ee053..9272714e30e 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H #define GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H -#include "src/core/lib/tsi/transport_security_interface.h" +#include "src/core/tsi/transport_security_interface.h" #ifdef __cplusplus extern "C" { diff --git a/test/core/bad_ssl/servers/cert.c b/test/core/bad_ssl/servers/cert.c index 9aadf452e2c..71a636c599d 100644 --- a/test/core/bad_ssl/servers/cert.c +++ b/test/core/bad_ssl/servers/cert.c @@ -56,10 +56,10 @@ int main(int argc, char **argv) { grpc_init(); GPR_ASSERT(GRPC_LOG_IF_ERROR( - "load_file", grpc_load_file("src/core/lib/tsi/test_creds/badserver.pem", + "load_file", grpc_load_file("src/core/tsi/test_creds/badserver.pem", 1, &cert_slice))); GPR_ASSERT(GRPC_LOG_IF_ERROR( - "load_file", grpc_load_file("src/core/lib/tsi/test_creds/badserver.key", + "load_file", grpc_load_file("src/core/tsi/test_creds/badserver.key", 1, &key_slice))); pem_key_cert_pair.private_key = (const char *)GRPC_SLICE_START_PTR(key_slice); pem_key_cert_pair.cert_chain = (const char *)GRPC_SLICE_START_PTR(cert_slice); diff --git a/test/core/handshake/BUILD b/test/core/handshake/BUILD index eb8f3a9beb9..996b503d353 100644 --- a/test/core/handshake/BUILD +++ b/test/core/handshake/BUILD @@ -34,9 +34,9 @@ cc_test( srcs = ["client_ssl.c"], copts = ["-std=c99"], data = [ - "//src/core/lib/tsi/test_creds:ca.pem", - "//src/core/lib/tsi/test_creds:server1.key", - "//src/core/lib/tsi/test_creds:server1.pem", + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", ], deps = [ "//:gpr", @@ -51,9 +51,9 @@ cc_test( srcs = ["server_ssl.c"], copts = ["-std=c99"], data = [ - "//src/core/lib/tsi/test_creds:ca.pem", - "//src/core/lib/tsi/test_creds:server1.key", - "//src/core/lib/tsi/test_creds:server1.pem", + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", ], deps = [ "//:gpr", diff --git a/test/core/handshake/client_ssl.c b/test/core/handshake/client_ssl.c index 5cfe60de4be..f291d09493b 100644 --- a/test/core/handshake/client_ssl.c +++ b/test/core/handshake/client_ssl.c @@ -54,9 +54,9 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -#define SSL_CERT_PATH "src/core/lib/tsi/test_creds/server1.pem" -#define SSL_KEY_PATH "src/core/lib/tsi/test_creds/server1.key" -#define SSL_CA_PATH "src/core/lib/tsi/test_creds/ca.pem" +#define SSL_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SSL_KEY_PATH "src/core/tsi/test_creds/server1.key" +#define SSL_CA_PATH "src/core/tsi/test_creds/ca.pem" // Arguments for TLS server thread. typedef struct { @@ -146,7 +146,7 @@ static int alpn_select_cb(SSL *ssl, const uint8_t **out, uint8_t *out_len, // Minimal TLS server. This is largely based on the example at // https://wiki.openssl.org/index.php/Simple_TLS_Server and the gRPC core -// internals in src/core/lib/tsi/ssl_transport_security.c. +// internals in src/core/tsi/ssl_transport_security.c. static void server_thread(void *arg) { const server_args *args = (server_args *)arg; @@ -172,7 +172,7 @@ static void server_thread(void *arg) { } // Set the cipher list to match the one expressed in - // src/core/lib/tsi/ssl_transport_security.c. + // src/core/tsi/ssl_transport_security.c. const char *cipher_list = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-" "SHA384:ECDHE-RSA-AES256-GCM-SHA384"; diff --git a/test/core/handshake/server_ssl.c b/test/core/handshake/server_ssl.c index 0bd5a03cffe..e568a370145 100644 --- a/test/core/handshake/server_ssl.c +++ b/test/core/handshake/server_ssl.c @@ -49,9 +49,9 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -#define SSL_CERT_PATH "src/core/lib/tsi/test_creds/server1.pem" -#define SSL_KEY_PATH "src/core/lib/tsi/test_creds/server1.key" -#define SSL_CA_PATH "src/core/lib/tsi/test_creds/ca.pem" +#define SSL_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SSL_KEY_PATH "src/core/tsi/test_creds/server1.key" +#define SSL_CA_PATH "src/core/tsi/test_creds/ca.pem" // Handshake completed signal to server thread. static gpr_event client_handshake_complete; @@ -174,7 +174,7 @@ static bool server_ssl_test(const char *alpn_list[], unsigned int alpn_list_len, } // Set the cipher list to match the one expressed in - // src/core/lib/tsi/ssl_transport_security.c. + // src/core/tsi/ssl_transport_security.c. const char *cipher_list = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-" "SHA384:ECDHE-RSA-AES256-GCM-SHA384"; diff --git a/test/core/http/test_server.py b/test/core/http/test_server.py index 86c2fe96bf9..dbbf5ceb3c7 100755 --- a/test/core/http/test_server.py +++ b/test/core/http/test_server.py @@ -36,8 +36,8 @@ import os import ssl import sys -_PEM = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..', 'src/core/lib/tsi/test_creds/server1.pem')) -_KEY = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..', 'src/core/lib/tsi/test_creds/server1.key')) +_PEM = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..', 'src/core/tsi/test_creds/server1.pem')) +_KEY = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..', 'src/core/tsi/test_creds/server1.key')) print _PEM open(_PEM).close() diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index b3775e91a72..8f11f98a9cd 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -43,7 +43,7 @@ #include "src/core/lib/iomgr/iomgr.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/tsi/fake_transport_security.h" +#include "src/core/tsi/fake_transport_security.h" #include "test/core/util/test_config.h" static gpr_mu *g_mu; diff --git a/test/core/security/security_connector_test.c b/test/core/security/security_connector_test.c index 4219b134f25..6c7b8e688f0 100644 --- a/test/core/security/security_connector_test.c +++ b/test/core/security/security_connector_test.c @@ -46,8 +46,8 @@ #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" #include "src/core/lib/support/tmpfile.h" -#include "src/core/lib/tsi/ssl_transport_security.h" -#include "src/core/lib/tsi/transport_security.h" +#include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" #include "test/core/util/test_config.h" static int check_transport_security_type(const grpc_auth_context *ctx) { diff --git a/test/core/surface/server_chttp2_test.c b/test/core/surface/server_chttp2_test.c index 6c178abdad1..8d40c646548 100644 --- a/test/core/surface/server_chttp2_test.c +++ b/test/core/surface/server_chttp2_test.c @@ -39,7 +39,7 @@ #include #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" -#include "src/core/lib/tsi/fake_transport_security.h" +#include "src/core/tsi/fake_transport_security.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index 01e8770b240..ee4a37c3141 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/tsi/transport_security.h" +#include "src/core/tsi/transport_security.h" #include @@ -43,8 +43,8 @@ #include #include "src/core/lib/support/string.h" -#include "src/core/lib/tsi/fake_transport_security.h" -#include "src/core/lib/tsi/ssl_transport_security.h" +#include "src/core/tsi/fake_transport_security.h" +#include "src/core/tsi/ssl_transport_security.h" #include "test/core/util/test_config.h" typedef struct { diff --git a/tools/http2_interop/http2interop_test.go b/tools/http2_interop/http2interop_test.go index 305125f0c11..fb314da1964 100644 --- a/tools/http2_interop/http2interop_test.go +++ b/tools/http2_interop/http2interop_test.go @@ -49,7 +49,7 @@ func InteropCtx(t *testing.T) *HTTP2InteropCtx { if ctx.UseTestCa { // It would be odd if useTestCa was true, but not useTls. meh - certData, err := ioutil.ReadFile("src/core/lib/tsi/test_creds/ca.pem") + certData, err := ioutil.ReadFile("src/core/tsi/test_creds/ca.pem") if err != nil { t.Fatal(err) } diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index fa21743f28d..00dd4ca6056 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -247,7 +247,7 @@ class CLanguage(object): polling_strategies = ['all'] for polling_strategy in polling_strategies: env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': - _ROOT + '/src/core/lib/tsi/test_creds/ca.pem', + _ROOT + '/src/core/tsi/test_creds/ca.pem', 'GRPC_POLL_STRATEGY': polling_strategy, 'GRPC_VERBOSITY': 'DEBUG'} resolver = os.environ.get('GRPC_DNS_RESOLVER', None); From 0633c4a8d7c96b332e23427ecfe9426baeeb9989 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 28 Mar 2017 12:34:04 -0700 Subject: [PATCH 050/105] Do not call grpc_init() for per-call-completion-queues created by a C++ synchronous server --- include/grpc++/impl/codegen/grpc_library.h | 26 ++++++++++++++-------- src/cpp/common/completion_queue_cc.cc | 3 ++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/include/grpc++/impl/codegen/grpc_library.h b/include/grpc++/impl/codegen/grpc_library.h index 2b11aff2149..01903ca18e4 100644 --- a/include/grpc++/impl/codegen/grpc_library.h +++ b/include/grpc++/impl/codegen/grpc_library.h @@ -51,18 +51,26 @@ extern GrpcLibraryInterface* g_glip; /// Classes that require gRPC to be initialized should inherit from this class. class GrpcLibraryCodegen { public: - GrpcLibraryCodegen() { - GPR_CODEGEN_ASSERT(g_glip && - "gRPC library not initialized. See " - "grpc::internal::GrpcLibraryInitializer."); - g_glip->init(); + GrpcLibraryCodegen(bool call_grpc_init = true) : grpc_init_called(false) { + if (call_grpc_init) { + GPR_CODEGEN_ASSERT(g_glip && + "gRPC library not initialized. See " + "grpc::internal::GrpcLibraryInitializer."); + g_glip->init(); + grpc_init_called = true; + } } virtual ~GrpcLibraryCodegen() { - GPR_CODEGEN_ASSERT(g_glip && - "gRPC library not initialized. See " - "grpc::internal::GrpcLibraryInitializer."); - g_glip->shutdown(); + if (grpc_init_called) { + GPR_CODEGEN_ASSERT(g_glip && + "gRPC library not initialized. See " + "grpc::internal::GrpcLibraryInitializer."); + g_glip->shutdown(); + } } + + private: + bool grpc_init_called; }; } // namespace grpc diff --git a/src/cpp/common/completion_queue_cc.cc b/src/cpp/common/completion_queue_cc.cc index 0408a410853..e7bb0cdd9a6 100644 --- a/src/cpp/common/completion_queue_cc.cc +++ b/src/cpp/common/completion_queue_cc.cc @@ -43,7 +43,8 @@ namespace grpc { static internal::GrpcLibraryInitializer g_gli_initializer; -CompletionQueue::CompletionQueue(grpc_completion_queue* take) : cq_(take) { +CompletionQueue::CompletionQueue(grpc_completion_queue* take) + : GrpcLibraryCodegen(false), cq_(take) { InitialAvalanching(); } From ef8e00230e46717467a6d8b308f2bda8ba9901d9 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 28 Mar 2017 12:34:04 -0700 Subject: [PATCH 051/105] Do not call grpc_init() for per-call-completion-queues created by a C++ synchronous server --- include/grpc++/impl/codegen/grpc_library.h | 26 ++++++++++++++-------- src/cpp/common/completion_queue_cc.cc | 3 ++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/include/grpc++/impl/codegen/grpc_library.h b/include/grpc++/impl/codegen/grpc_library.h index 2b11aff2149..3735d04e8c1 100644 --- a/include/grpc++/impl/codegen/grpc_library.h +++ b/include/grpc++/impl/codegen/grpc_library.h @@ -51,18 +51,26 @@ extern GrpcLibraryInterface* g_glip; /// Classes that require gRPC to be initialized should inherit from this class. class GrpcLibraryCodegen { public: - GrpcLibraryCodegen() { - GPR_CODEGEN_ASSERT(g_glip && - "gRPC library not initialized. See " - "grpc::internal::GrpcLibraryInitializer."); - g_glip->init(); + GrpcLibraryCodegen(bool call_grpc_init = true) : grpc_init_called_(false) { + if (call_grpc_init) { + GPR_CODEGEN_ASSERT(g_glip && + "gRPC library not initialized. See " + "grpc::internal::GrpcLibraryInitializer."); + g_glip->init(); + grpc_init_called_ = true; + } } virtual ~GrpcLibraryCodegen() { - GPR_CODEGEN_ASSERT(g_glip && - "gRPC library not initialized. See " - "grpc::internal::GrpcLibraryInitializer."); - g_glip->shutdown(); + if (grpc_init_called_) { + GPR_CODEGEN_ASSERT(g_glip && + "gRPC library not initialized. See " + "grpc::internal::GrpcLibraryInitializer."); + g_glip->shutdown(); + } } + + private: + bool grpc_init_called_; }; } // namespace grpc diff --git a/src/cpp/common/completion_queue_cc.cc b/src/cpp/common/completion_queue_cc.cc index 0408a410853..e7bb0cdd9a6 100644 --- a/src/cpp/common/completion_queue_cc.cc +++ b/src/cpp/common/completion_queue_cc.cc @@ -43,7 +43,8 @@ namespace grpc { static internal::GrpcLibraryInitializer g_gli_initializer; -CompletionQueue::CompletionQueue(grpc_completion_queue* take) : cq_(take) { +CompletionQueue::CompletionQueue(grpc_completion_queue* take) + : GrpcLibraryCodegen(false), cq_(take) { InitialAvalanching(); } From 996a6734235ac21cb6a6222425fadc93706c93c3 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 28 Mar 2017 18:34:52 -0700 Subject: [PATCH 052/105] minor changes to get docker building on osx docker on osx needs slightly different command line option --- tools/run_tests/dockerize/build_interop_image.sh | 8 +++++++- tools/run_tests/run_interop_tests.py | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/dockerize/build_interop_image.sh b/tools/run_tests/dockerize/build_interop_image.sh index 48a216a124b..3385738f9c3 100755 --- a/tools/run_tests/dockerize/build_interop_image.sh +++ b/tools/run_tests/dockerize/build_interop_image.sh @@ -71,7 +71,13 @@ then fi # Use image name based on Dockerfile checksum -BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/interoptest/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +# on OSX use md5 instead of sha1sum +if which sha1sum > /dev/null; +then + BASE_IMAGE=${BASE_NAME}_base:`sha1sum tools/dockerfile/interoptest/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +else + BASE_IMAGE=${BASE_NAME}_base:`md5 -r tools/dockerfile/interoptest/$BASE_NAME/Dockerfile | cut -f1 -d\ ` +fi # Make sure base docker image has been built. Should be instantaneous if so. docker build -t $BASE_IMAGE --force-rm=true tools/dockerfile/interoptest/$BASE_NAME || exit $? diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 2d7f4a625d6..1a83c10a42c 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -728,7 +728,8 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): ] else: - docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] + portstr = str(_DEFAULT_SERVER_PORT) + docker_args += ['-p', '%s:%s'%(portstr, portstr)] docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, From 26cb316227d6b05bac2d083d603bffc8d90f55d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 19:31:20 -0700 Subject: [PATCH 053/105] fix typo --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 332f3be4f8a..cafb3c1076b 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -82,7 +82,7 @@ argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_ argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=5) argp.add_argument('-p', '--p_threshold', type=float, default=0.05) -args.add_argument('-g', '--git_comment', action='store_const', const=True, default=False) +argp.add_argument('-g', '--git_comment', action='store_const', const=True, default=False) args = argp.parse_args() assert args.diff_base From 1aba869e9e3046766cab8744a028a20a8af62dd2 Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 28 Mar 2017 21:41:02 -0700 Subject: [PATCH 054/105] Make convert clock_type consistent with add/sub when dealing with extreme values --- src/core/lib/support/time.c | 12 +++--------- test/core/support/time_test.c | 9 +++++++++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/core/lib/support/time.c b/src/core/lib/support/time.c index 5a7d043aed8..614a068ddd3 100644 --- a/src/core/lib/support/time.c +++ b/src/core/lib/support/time.c @@ -244,15 +244,9 @@ gpr_timespec gpr_convert_clock_type(gpr_timespec t, gpr_clock_type clock_type) { return t; } - if (t.tv_nsec == 0) { - if (t.tv_sec == INT64_MAX) { - t.clock_type = clock_type; - return t; - } - if (t.tv_sec == INT64_MIN) { - t.clock_type = clock_type; - return t; - } + if (t.tv_sec == INT64_MAX || t.tv_sec == INT64_MIN) { + t.clock_type = clock_type; + return t; } if (clock_type == GPR_TIMESPAN) { diff --git a/test/core/support/time_test.c b/test/core/support/time_test.c index e9ca08d0419..73e43caa696 100644 --- a/test/core/support/time_test.c +++ b/test/core/support/time_test.c @@ -255,6 +255,14 @@ static void test_similar(void) { gpr_time_from_micros(10, GPR_TIMESPAN))); } +static void test_convert_extreme(void) { + gpr_timespec realtime_t = {INT64_MAX, 1, GPR_CLOCK_REALTIME}; + gpr_timespec monotime_t = + gpr_convert_clock_type(realtime_t, GPR_CLOCK_MONOTONIC); + GPR_ASSERT(monotime_t.tv_sec == realtime_t.tv_sec); + GPR_ASSERT(monotime_t.clock_type == GPR_CLOCK_MONOTONIC); +} + int main(int argc, char *argv[]) { grpc_test_init(argc, argv); @@ -263,5 +271,6 @@ int main(int argc, char *argv[]) { test_overflow(); test_sticky_infinities(); test_similar(); + test_convert_extreme(); return 0; } From ba32fee3fc2e054ef87148d9027fc08c8c8c37c1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 01:49:02 -0700 Subject: [PATCH 055/105] Speed up builds --- tools/profiling/microbenchmarks/bm_diff.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index cafb3c1076b..ffbdfeb77fd 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -95,12 +95,17 @@ def avg(lst): n += 1 return sum / n -def collect1(bm, cfg, ver): - subprocess.check_call(['make', 'clean']) +def build(): subprocess.check_call(['git', 'submodule', 'update']) - subprocess.check_call( - ['make', bm, - 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) + make = ['make'] + args.benchmarks + [ + 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()] + try: + subprocess.check_call(make) + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + subprocess.check_call(make) + +def collect1(bm, cfg, ver): cmd = ['bins/%s/%s' % (cfg, bm), '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver), '--benchmark_out_format=json', @@ -108,6 +113,7 @@ def collect1(bm, cfg, ver): ] subprocess.check_call(cmd) +build() for bm in args.benchmarks: collect1(bm, 'opt', 'new') collect1(bm, 'counters', 'new') @@ -116,6 +122,7 @@ where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD' subprocess.check_call(['git', 'checkout', args.diff_base]) try: + build() comparables = [] for bm in args.benchmarks: try: From d1d055a1bf149b0e971dcc3290a2ccfc346217ee Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 01:52:28 -0700 Subject: [PATCH 056/105] Fix building --- tools/profiling/microbenchmarks/bm_diff.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index ffbdfeb77fd..7281bd77e77 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -95,15 +95,19 @@ def avg(lst): n += 1 return sum / n -def build(): - subprocess.check_call(['git', 'submodule', 'update']) +def make_cmd(cfg): make = ['make'] + args.benchmarks + [ 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()] + +def build(): + subprocess.check_call(['git', 'submodule', 'update']) try: - subprocess.check_call(make) + subprocess.check_call(make_cmd('opt')) + subprocess.check_call(make_cmd('counters')) except subprocess.CalledProcessError, e: subprocess.check_call(['make', 'clean']) - subprocess.check_call(make) + subprocess.check_call(make_cmd('opt')) + subprocess.check_call(make_cmd('counters')) def collect1(bm, cfg, ver): cmd = ['bins/%s/%s' % (cfg, bm), From fda22624bae8b5911ebdc35d2c938476d92e933e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 01:54:16 -0700 Subject: [PATCH 057/105] Fix building --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 7281bd77e77..b74a857ade8 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -96,7 +96,7 @@ def avg(lst): return sum / n def make_cmd(cfg): - make = ['make'] + args.benchmarks + [ + return ['make'] + args.benchmarks + [ 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()] def build(): From ebb7e82c73884511188d3f43f5e9df1d6058aa66 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:17:11 -0700 Subject: [PATCH 058/105] Use python for commenting --- tools/profiling/microbenchmarks/bm_diff.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index b74a857ade8..43ba1b47e05 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -38,6 +38,9 @@ import subprocess import multiprocessing import collections import pipes +import os +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) +import comment_on_pr def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -82,7 +85,6 @@ argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_ argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=5) argp.add_argument('-p', '--p_threshold', type=float, default=0.05) -argp.add_argument('-g', '--git_comment', action='store_const', const=True, default=False) args = argp.parse_args() assert args.diff_base @@ -206,8 +208,6 @@ if rows: text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') else: text = 'No significant performance differences' +comment_on_pr.comment_on_pr(text) print text -if args.git_comment: - subprocess.call(['tools/jenkins/comment_on_pr.sh', text], - stdout=subprocess.PIPE) From 6a17fddc6996d555d508cbda50b990ca2bca3c82 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:21:10 -0700 Subject: [PATCH 059/105] Threshold % change also --- tools/profiling/microbenchmarks/bm_diff.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 43ba1b47e05..84217ec40ba 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -161,8 +161,11 @@ class Benchmark: old = self.samples[False][f] if not new or not old: continue p = stats.ttest_ind(new, old)[1] - delta = avg(new) - avg(old) - if p < args.p_threshold and abs(delta) > 0.1: + new_avg = avg(new) + old_avg = avg(old) + delta = new_avg - old_avg + ratio = changed_ratio(new_avg, old_avg) + if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.05: self.final[f] = delta return self.final.keys() @@ -210,4 +213,3 @@ else: text = 'No significant performance differences' comment_on_pr.comment_on_pr(text) print text - From 57a06bfd15a3426267ae656d4cffa079f2edcca9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:31:30 -0700 Subject: [PATCH 060/105] Always re-update submodules --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 84217ec40ba..fa4cbf252da 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -139,6 +139,7 @@ try: pass finally: subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) class Benchmark: From 5431f909601ee4eb4984aed24b6614a324bed76e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:31:45 -0700 Subject: [PATCH 061/105] Add python commenter module --- tools/run_tests/python_utils/comment_on_pr.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tools/run_tests/python_utils/comment_on_pr.py diff --git a/tools/run_tests/python_utils/comment_on_pr.py b/tools/run_tests/python_utils/comment_on_pr.py new file mode 100644 index 00000000000..7f39671659d --- /dev/null +++ b/tools/run_tests/python_utils/comment_on_pr.py @@ -0,0 +1,50 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import urllib2 + +def comment_on_pr(text): + if 'JENKINS_OAUTH_TOKEN' not in os.environ: + print 'Missing JENKINS_OAUTH_TOKEN env var: not commenting' + return + if 'ghprbPullId' not in os.environ: + print 'Missing ghprbPullId env var: not commenting' + return + req = urllib2.Request( + url = 'https://api.github.com/repos/grpc/grpc/issues/%s/comments' % + os.environ('ghprbPullId'), + data = json.dumps({'body': text}), + headers = { + 'Authorization': 'token %s' % os.environ['JENKINS_OAUTH_TOKEN'], + 'Content-Type': 'application/json', + }) + with urllib2.urlopen(req) as rsp: + print rsp.read() + From ab1c6c3b313d7e3ef641b23f1faba641b8044b0c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:33:27 -0700 Subject: [PATCH 062/105] Cleanup args --- tools/jenkins/run_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index c2b7af063f8..87699a283e7 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -38,6 +38,6 @@ BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_c cd $(dirname $0)/../.. # todo(mattkwong): Change performance test to use microbenchmarking -tools/profiling/microbenchmarks/bm_diff.py -g -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN +tools/profiling/microbenchmarks/bm_diff.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN tools/run_tests/run_performance_tests.py -l c++ node ruby csharp python --netperf --category smoketest From 730ddc275816c56b0e304ed3aba10505f6901178 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:38:47 -0700 Subject: [PATCH 063/105] Sanity fixes --- src/core/lib/transport/bdp_estimator.h | 2 +- src/core/tsi/fake_transport_security.h | 6 +++--- src/core/tsi/ssl_transport_security.h | 6 +++--- src/core/tsi/ssl_types.h | 6 +++--- src/core/tsi/transport_security.h | 6 +++--- src/core/tsi/transport_security_interface.h | 6 +++--- test/core/bad_ssl/servers/cert.c | 8 ++++---- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h index bcaf899910d..df8d1f6fc0f 100644 --- a/src/core/lib/transport/bdp_estimator.h +++ b/src/core/lib/transport/bdp_estimator.h @@ -73,4 +73,4 @@ void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator); // Completes a previously started ping void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator); -#endif +#endif /* GRPC_CORE_LIB_TRANSPORT_BDP_ESTIMATOR_H */ diff --git a/src/core/tsi/fake_transport_security.h b/src/core/tsi/fake_transport_security.h index 7c1c8ed195c..0697c7279f1 100644 --- a/src/core/tsi/fake_transport_security.h +++ b/src/core/tsi/fake_transport_security.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H -#define GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H +#ifndef GRPC_CORE_TSI_FAKE_TRANSPORT_SECURITY_H +#define GRPC_CORE_TSI_FAKE_TRANSPORT_SECURITY_H #include "src/core/tsi/transport_security_interface.h" @@ -58,4 +58,4 @@ tsi_frame_protector *tsi_create_fake_protector( } #endif -#endif /* GRPC_CORE_LIB_TSI_FAKE_TRANSPORT_SECURITY_H */ +#endif /* GRPC_CORE_TSI_FAKE_TRANSPORT_SECURITY_H */ diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index afc25da98d2..0a527e9021f 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H -#define GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H +#ifndef GRPC_CORE_TSI_SSL_TRANSPORT_SECURITY_H +#define GRPC_CORE_TSI_SSL_TRANSPORT_SECURITY_H #include "src/core/tsi/transport_security_interface.h" @@ -188,4 +188,4 @@ int tsi_ssl_peer_matches_name(const tsi_peer *peer, const char *name); } #endif -#endif /* GRPC_CORE_LIB_TSI_SSL_TRANSPORT_SECURITY_H */ +#endif /* GRPC_CORE_TSI_SSL_TRANSPORT_SECURITY_H */ diff --git a/src/core/tsi/ssl_types.h b/src/core/tsi/ssl_types.h index 0a988effd0f..065cb868000 100644 --- a/src/core/tsi/ssl_types.h +++ b/src/core/tsi/ssl_types.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_TSI_SSL_TYPES_H -#define GRPC_CORE_LIB_TSI_SSL_TYPES_H +#ifndef GRPC_CORE_TSI_SSL_TYPES_H +#define GRPC_CORE_TSI_SSL_TYPES_H /* A collection of macros to cast between various integer types that are * used differently between BoringSSL and OpenSSL: @@ -52,4 +52,4 @@ #define TSI_SIZE_AS_SIZE(x) ((int)(x)) #endif -#endif /* GRPC_CORE_LIB_TSI_SSL_TYPES_H */ +#endif /* GRPC_CORE_TSI_SSL_TYPES_H */ diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index 9272714e30e..491fa1a8bd6 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H -#define GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H +#ifndef GRPC_CORE_TSI_TRANSPORT_SECURITY_H +#define GRPC_CORE_TSI_TRANSPORT_SECURITY_H #include "src/core/tsi/transport_security_interface.h" @@ -108,4 +108,4 @@ char *tsi_strdup(const char *src); /* Sadly, no strdup in C89. */ } #endif -#endif /* GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_H */ +#endif /* GRPC_CORE_TSI_TRANSPORT_SECURITY_H */ diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index 3e8c9d7ffef..caed43eac44 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_INTERFACE_H -#define GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_INTERFACE_H +#ifndef GRPC_CORE_TSI_TRANSPORT_SECURITY_INTERFACE_H +#define GRPC_CORE_TSI_TRANSPORT_SECURITY_INTERFACE_H #include #include @@ -350,4 +350,4 @@ void tsi_handshaker_destroy(tsi_handshaker *self); } #endif -#endif /* GRPC_CORE_LIB_TSI_TRANSPORT_SECURITY_INTERFACE_H */ +#endif /* GRPC_CORE_TSI_TRANSPORT_SECURITY_INTERFACE_H */ diff --git a/test/core/bad_ssl/servers/cert.c b/test/core/bad_ssl/servers/cert.c index 71a636c599d..2b014b23431 100644 --- a/test/core/bad_ssl/servers/cert.c +++ b/test/core/bad_ssl/servers/cert.c @@ -56,11 +56,11 @@ int main(int argc, char **argv) { grpc_init(); GPR_ASSERT(GRPC_LOG_IF_ERROR( - "load_file", grpc_load_file("src/core/tsi/test_creds/badserver.pem", - 1, &cert_slice))); + "load_file", + grpc_load_file("src/core/tsi/test_creds/badserver.pem", 1, &cert_slice))); GPR_ASSERT(GRPC_LOG_IF_ERROR( - "load_file", grpc_load_file("src/core/tsi/test_creds/badserver.key", - 1, &key_slice))); + "load_file", + grpc_load_file("src/core/tsi/test_creds/badserver.key", 1, &key_slice))); pem_key_cert_pair.private_key = (const char *)GRPC_SLICE_START_PTR(key_slice); pem_key_cert_pair.cert_chain = (const char *)GRPC_SLICE_START_PTR(cert_slice); From 39311f5599f1206e67c1b616e81b8e4d2afed1a2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 09:25:06 -0700 Subject: [PATCH 064/105] Fix comments --- src/core/lib/iomgr/ev_epoll_linux.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index c4c67df381c..97f8f82fed9 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1135,7 +1135,7 @@ static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, is no other code that needs to 'happen-after' this) */ if (gpr_atm_no_barrier_cas(state, CLOSURE_READY, CLOSURE_NOT_READY)) { grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); - return; /* Slow-path successful. Return */ + return; /* Successful. Return */ } break; /* retry */ @@ -1169,8 +1169,6 @@ static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, gpr_atm new_state = (gpr_atm)shutdown_err | FD_SHUTDOWN_BIT; while (true) { - /* Fallback to slowpath. This 'acquire' load matches the 'release' cas in - notify_on and set_ready */ gpr_atm curr = gpr_atm_no_barrier_load(state); switch (curr) { case CLOSURE_READY: From a61dce07a860e6d91ce65b88e3ad9468014bb979 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 09:30:07 -0700 Subject: [PATCH 065/105] Syntax fix --- tools/run_tests/python_utils/comment_on_pr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/run_tests/python_utils/comment_on_pr.py b/tools/run_tests/python_utils/comment_on_pr.py index 7f39671659d..edd3603f67d 100644 --- a/tools/run_tests/python_utils/comment_on_pr.py +++ b/tools/run_tests/python_utils/comment_on_pr.py @@ -39,7 +39,7 @@ def comment_on_pr(text): return req = urllib2.Request( url = 'https://api.github.com/repos/grpc/grpc/issues/%s/comments' % - os.environ('ghprbPullId'), + os.environ['ghprbPullId'], data = json.dumps({'body': text}), headers = { 'Authorization': 'token %s' % os.environ['JENKINS_OAUTH_TOKEN'], @@ -47,4 +47,3 @@ def comment_on_pr(text): }) with urllib2.urlopen(req) as rsp: print rsp.read() - From 5c0ef3667961be2a3d09f1c019fd6cef2777dcd9 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 29 Mar 2017 09:53:48 -0700 Subject: [PATCH 066/105] Add comment --- src/cpp/common/completion_queue_cc.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cpp/common/completion_queue_cc.cc b/src/cpp/common/completion_queue_cc.cc index e7bb0cdd9a6..14c51f63c5c 100644 --- a/src/cpp/common/completion_queue_cc.cc +++ b/src/cpp/common/completion_queue_cc.cc @@ -43,6 +43,10 @@ namespace grpc { static internal::GrpcLibraryInitializer g_gli_initializer; +// 'CompletionQueue' constructor can safely call GrpcLibraryCodegen(false) here +// i.e not have GrpcLibraryCodegen call grpc_init(). This is because, to create +// a 'grpc_completion_queue' instance (which is being passed as the input to +// this constructor), one must have already called grpc_init(). CompletionQueue::CompletionQueue(grpc_completion_queue* take) : GrpcLibraryCodegen(false), cq_(take) { InitialAvalanching(); From ea9d8b67757526164c0b4cb86f61d2322d30f62d Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 29 Mar 2017 10:20:19 -0700 Subject: [PATCH 067/105] Display time between tests for troubleshooting timeout --- src/objective-c/tests/build_example_test.sh | 9 +++++++++ src/objective-c/tests/build_tests.sh | 1 + src/objective-c/tests/run_tests.sh | 6 +++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/objective-c/tests/build_example_test.sh b/src/objective-c/tests/build_example_test.sh index ae75941ec61..5c50e831107 100755 --- a/src/objective-c/tests/build_example_test.sh +++ b/src/objective-c/tests/build_example_test.sh @@ -35,29 +35,38 @@ set -evo pipefail cd `dirname $0` +trap 'echo "EXIT TIME: $(date)"' EXIT + +echo "TIME: $(date)" SCHEME=HelloWorld \ EXAMPLE_PATH=examples/objective-c/helloworld \ ./build_one_example.sh +echo "TIME: $(date)" SCHEME=RouteGuideClient \ EXAMPLE_PATH=examples/objective-c/route_guide \ ./build_one_example.sh +echo "TIME: $(date)" SCHEME=AuthSample \ EXAMPLE_PATH=examples/objective-c/auth_sample \ ./build_one_example.sh rm -f ../examples/RemoteTestClient/*.{h,m} +echo "TIME: $(date)" SCHEME=Sample \ EXAMPLE_PATH=src/objective-c/examples/Sample \ ./build_one_example.sh +echo "TIME: $(date)" SCHEME=Sample \ EXAMPLE_PATH=src/objective-c/examples/Sample \ FRAMEWORKS=YES \ ./build_one_example.sh +echo "TIME: $(date)" SCHEME=SwiftSample \ EXAMPLE_PATH=src/objective-c/examples/SwiftSample \ ./build_one_example.sh + diff --git a/src/objective-c/tests/build_tests.sh b/src/objective-c/tests/build_tests.sh index bc5bc044943..6602d510d94 100755 --- a/src/objective-c/tests/build_tests.sh +++ b/src/objective-c/tests/build_tests.sh @@ -50,4 +50,5 @@ rm -rf Tests.xcworkspace rm -f Podfile.lock rm -f RemoteTestClient/*.{h,m} +echo "TIME: $(date)" pod install diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index d217f1c8e02..bd7c2945a27 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -47,31 +47,35 @@ BINDIR=../../../bins/$CONFIG $BINDIR/interop_server --port=5050 --max_send_message_size=8388608 & $BINDIR/interop_server --port=5051 --max_send_message_size=8388608 --use_tls & # Kill them when this script exits. -trap 'kill -9 `jobs -p`' EXIT +trap 'kill -9 `jobs -p` ; echo "EXIT TIME: $(date)"' EXIT # xcodebuild is very verbose. We filter its output and tell Bash to fail if any # element of the pipe fails. # TODO(jcanizales): Use xctool instead? Issue #2540. set -o pipefail XCODEBUILD_FILTER='(^===|^\*\*|\bfatal\b|\berror\b|\bwarning\b|\bfail)' +echo "TIME: $(date)" xcodebuild \ -workspace Tests.xcworkspace \ -scheme AllTests \ -destination name="iPhone 6" \ test | xcpretty +echo "TIME: $(date)" xcodebuild \ -workspace Tests.xcworkspace \ -scheme CoreCronetEnd2EndTests \ -destination name="iPhone 6" \ test | xcpretty +echo "TIME: $(date)" xcodebuild \ -workspace Tests.xcworkspace \ -scheme CronetUnitTests \ -destination name="iPhone 6" \ test | xcpretty +echo "TIME: $(date)" xcodebuild \ -workspace Tests.xcworkspace \ -scheme InteropTestsRemoteWithCronet \ From aab6992c006be6fb80df73fd9f218365099c016d Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 28 Mar 2017 22:58:56 -0700 Subject: [PATCH 068/105] Remove call from sibling list when it is not first child. --- src/core/lib/surface/call.c | 4 +-- .../clusterfuzz-testcase-5965570207907840 | Bin 0 -> 1883 bytes tools/run_tests/generated/tests.json | 23 ++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5965570207907840 diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 895a8a3b060..4f5e815dab2 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -504,9 +504,9 @@ void grpc_call_destroy(grpc_call *c) { if (c == parent->first_child) { parent->first_child = NULL; } - c->sibling_prev->sibling_next = c->sibling_next; - c->sibling_next->sibling_prev = c->sibling_prev; } + c->sibling_prev->sibling_next = c->sibling_next; + c->sibling_next->sibling_prev = c->sibling_prev; gpr_mu_unlock(&parent->child_list_mu); GRPC_CALL_INTERNAL_UNREF(&exec_ctx, parent, "child"); } diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5965570207907840 b/test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5965570207907840 new file mode 100644 index 0000000000000000000000000000000000000000..a58dc7b2442fc21b5bbdf2de959c4e2625408e6b GIT binary patch literal 1883 zcmeH|y-ve05XaAMWumrXL1OO61RF~S9)bsmqSkGZ+9q;LGQyCBiAP{$>KpJhJO&Kr z?1a|&vJzkBSy5 zbNSRV3=M#+L^QvQ}426#jsGJOSXy3BX8O zNNC9wLw>{k8hzM{+kgL+&e-*cPP0y8aYs&oHR)va2!MsOnl<`)hwxhaoy{7&C3l<< zOb0s$yT*stQ?S{Llio8{RSQmbTkNWkQr}7J>8+F1=H1Wl^*^)h^zR=lYBY7hm%;r; z`tw|9o0Zx~=UU3BS;;T7x@sZUQXH3Mddq#S(nh2|$4%dB)8Jo?4KUk}th`7J2+W39 zkN}_bx92S-XcrXzn*$t=RxL*Mstg{%Bf^`)dj<*%k(dI5yaZtfC>qZ8ddXS_k*Q^+ WOFB4-E~1kFl$lzF7hD86e=}cRIc7Zo literal 0 HcmV?d00001 diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 87142f585ce..ca5d7a7a33a 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -82088,6 +82088,29 @@ ], "uses_polling": false }, + { + "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5965570207907840" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [ + "tsan" + ], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "mac", + "linux" + ], + "uses_polling": false + }, { "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-6520142139752448" From 8dbd9677ef2a2b7b23a864366e37c7856a89a2ca Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 29 Mar 2017 10:58:21 -0700 Subject: [PATCH 069/105] Add a benchmark --- test/cpp/microbenchmarks/bm_cq.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc index 91e6a851017..55eec84b3ba 100644 --- a/test/cpp/microbenchmarks/bm_cq.cc +++ b/test/cpp/microbenchmarks/bm_cq.cc @@ -59,6 +59,17 @@ static void BM_CreateDestroyCpp(benchmark::State& state) { } BENCHMARK(BM_CreateDestroyCpp); +/* Create cq using a different constructor */ +static void BM_CreateDestroyCpp2(benchmark::State& state) { + TrackCounters track_counters; + while (state.KeepRunning()) { + grpc_completion_queue* core_cq = grpc_completion_queue_create(NULL); + CompletionQueue cq(core_cq); + } + track_counters.Finish(state); +} +BENCHMARK(BM_CreateDestroyCpp2); + static void BM_CreateDestroyCore(benchmark::State& state) { TrackCounters track_counters; while (state.KeepRunning()) { From 70abee26877782758f2105cdbc33fab8f0c0ade2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 12:19:34 -0700 Subject: [PATCH 070/105] Missing import --- tools/run_tests/python_utils/comment_on_pr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/comment_on_pr.py b/tools/run_tests/python_utils/comment_on_pr.py index edd3603f67d..5c29abb0192 100644 --- a/tools/run_tests/python_utils/comment_on_pr.py +++ b/tools/run_tests/python_utils/comment_on_pr.py @@ -28,6 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os +import json import urllib2 def comment_on_pr(text): From b0367ca2446cc61f697d5d3cf30b3ac5f4ae6ed8 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 29 Mar 2017 21:53:38 +0200 Subject: [PATCH 071/105] Fixing embedded build of cares for certain targets. --- Makefile | 48 ++++++++++++++++++------------------- templates/Makefile.template | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index bfc43aab28e..ef1732a820c 100644 --- a/Makefile +++ b/Makefile @@ -3081,7 +3081,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE else -$(LIBDIR)/$(CONFIG)/libgrpc.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(OPENSSL_MERGE_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBGRPC_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(OPENSSL_MERGE_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc.a @@ -3369,7 +3369,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_E else -$(LIBDIR)/$(CONFIG)/libgrpc_cronet.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_CRONET_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(OPENSSL_MERGE_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc_cronet.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBGRPC_CRONET_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(OPENSSL_MERGE_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_cronet.a @@ -3597,7 +3597,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a: openssl_dep_error else -$(LIBDIR)/$(CONFIG)/libgrpc_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_TEST_UTIL_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBGRPC_TEST_UTIL_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a @@ -3950,7 +3950,7 @@ $(LIBDIR)/$(CONFIG)/libreconnect_server.a: openssl_dep_error else -$(LIBDIR)/$(CONFIG)/libreconnect_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBRECONNECT_SERVER_OBJS) +$(LIBDIR)/$(CONFIG)/libreconnect_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBRECONNECT_SERVER_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libreconnect_server.a @@ -3989,7 +3989,7 @@ $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a: openssl_dep_error else -$(LIBDIR)/$(CONFIG)/libtest_tcp_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBTEST_TCP_SERVER_OBJS) +$(LIBDIR)/$(CONFIG)/libtest_tcp_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBTEST_TCP_SERVER_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a @@ -4175,7 +4175,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++$(SHARED_VERSION_CPP).$(SHARED_EXT_CPP else -$(LIBDIR)/$(CONFIG)/libgrpc++.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc++.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++.a @@ -4573,7 +4573,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_cronet$(SHARED_VERSION_CPP).$(SHARED_ else -$(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_CRONET_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(OPENSSL_MERGE_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_CRONET_OBJS) $(LIBGPR_OBJS) $(ZLIB_MERGE_OBJS) $(CARES_MERGE_OBJS) $(OPENSSL_MERGE_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a @@ -4641,7 +4641,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_PROTO_REFLECTION_DESC_DB_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_PROTO_REFLECTION_DESC_DB_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a @@ -4696,7 +4696,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc++_reflection$(SHARED_VERSION_CPP).$(SHA else -$(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_REFLECTION_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_REFLECTION_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a @@ -4764,7 +4764,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_TEST_CONFIG_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_TEST_CONFIG_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a @@ -4876,7 +4876,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_TEST_UTIL_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_TEST_UTIL_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a @@ -5116,7 +5116,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC_BENCHMARK_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC_BENCHMARK_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a @@ -5171,7 +5171,7 @@ $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC_CLI_LIBS_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBGRPC_CLI_LIBS_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a @@ -5271,7 +5271,7 @@ $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libhttp2_client_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBHTTP2_CLIENT_MAIN_OBJS) +$(LIBDIR)/$(CONFIG)/libhttp2_client_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBHTTP2_CLIENT_MAIN_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a @@ -5322,7 +5322,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libinterop_client_helper.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_CLIENT_HELPER_OBJS) +$(LIBDIR)/$(CONFIG)/libinterop_client_helper.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_CLIENT_HELPER_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a @@ -5376,7 +5376,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_client_main.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libinterop_client_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_CLIENT_MAIN_OBJS) +$(LIBDIR)/$(CONFIG)/libinterop_client_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_CLIENT_MAIN_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_client_main.a @@ -5427,7 +5427,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libinterop_server_helper.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_SERVER_HELPER_OBJS) +$(LIBDIR)/$(CONFIG)/libinterop_server_helper.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_SERVER_HELPER_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a @@ -5479,7 +5479,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libinterop_server_lib.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_SERVER_LIB_OBJS) +$(LIBDIR)/$(CONFIG)/libinterop_server_lib.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_SERVER_LIB_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a @@ -5529,7 +5529,7 @@ $(LIBDIR)/$(CONFIG)/libinterop_server_main.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libinterop_server_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_SERVER_MAIN_OBJS) +$(LIBDIR)/$(CONFIG)/libinterop_server_main.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBINTEROP_SERVER_MAIN_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libinterop_server_main.a @@ -5592,7 +5592,7 @@ $(LIBDIR)/$(CONFIG)/libqps.a: protobuf_dep_error else -$(LIBDIR)/$(CONFIG)/libqps.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBQPS_OBJS) +$(LIBDIR)/$(CONFIG)/libqps.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(PROTOBUF_DEP) $(LIBQPS_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libqps.a @@ -5644,7 +5644,7 @@ $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_csharp_ext$(SHARED_VERSION_CSHARP).$(SH else -$(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBGRPC_CSHARP_EXT_OBJS) +$(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBGRPC_CSHARP_EXT_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a @@ -7872,7 +7872,7 @@ $(LIBDIR)/$(CONFIG)/libbad_client_test.a: openssl_dep_error else -$(LIBDIR)/$(CONFIG)/libbad_client_test.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBBAD_CLIENT_TEST_OBJS) +$(LIBDIR)/$(CONFIG)/libbad_client_test.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBBAD_CLIENT_TEST_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libbad_client_test.a @@ -7911,7 +7911,7 @@ $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a: openssl_dep_error else -$(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBBAD_SSL_TEST_SERVER_OBJS) +$(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBBAD_SSL_TEST_SERVER_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a @@ -8001,7 +8001,7 @@ $(LIBDIR)/$(CONFIG)/libend2end_tests.a: openssl_dep_error else -$(LIBDIR)/$(CONFIG)/libend2end_tests.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(LIBEND2END_TESTS_OBJS) +$(LIBDIR)/$(CONFIG)/libend2end_tests.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(LIBEND2END_TESTS_OBJS) $(E) "[AR] Creating $@" $(Q) mkdir -p `dirname $@` $(Q) rm -f $(LIBDIR)/$(CONFIG)/libend2end_tests.a diff --git a/templates/Makefile.template b/templates/Makefile.template index 3ea69f8ca5d..018e276cca4 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -1477,7 +1477,7 @@ else % endif - $(LIBDIR)/$(CONFIG)/lib${lib.name}.a: $(ZLIB_DEP) $(OPENSSL_DEP)\ + $(LIBDIR)/$(CONFIG)/lib${lib.name}.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP)\ ## The else here corresponds to the if secure earlier. % else: % if lib.language == 'c++': From d51bbeb1ebc3afe6ce6f614eda3137fd39a70592 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 13:46:12 -0700 Subject: [PATCH 072/105] Fix leak --- test/core/iomgr/ev_epoll_linux_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 1a600e34d66..5f8124aedae 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -393,6 +393,7 @@ static void test_threading(void) { grpc_schedule_on_exec_ctx)); grpc_exec_ctx_finish(&exec_ctx); } + gpr_free(shared.pollset); } int main(int argc, char **argv) { From 61c317b624e14eaefceca257473e9df97e1ced60 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 13:56:24 -0700 Subject: [PATCH 073/105] Fix leak --- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index b61b325dfa9..8fbfd0fa132 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -147,9 +147,12 @@ template class SingleInternedBinaryElem { public: static std::vector GetElems(grpc_exec_ctx *exec_ctx) { - return {grpc_mdelem_from_slices( + grpc_slice bytes = MakeBytes(); + std::vector out = {grpc_mdelem_from_slices( exec_ctx, grpc_slice_intern(grpc_slice_from_static_string("abc-bin")), - grpc_slice_intern(MakeBytes()))}; + grpc_slice_intern(bytes))}; + grpc_slice_unref(bytes); + return out; } private: @@ -325,6 +328,8 @@ static void BM_HpackParserParseHeader(benchmark::State &state) { } grpc_exec_ctx_flush(&exec_ctx); } + for (auto slice : init_slices) grpc_slice_unref(slice); + for (auto slice : benchmark_slices) grpc_slice_unref(slice); grpc_chttp2_hpack_parser_destroy(&exec_ctx, &p); grpc_exec_ctx_finish(&exec_ctx); track_counters.Finish(state); From 8a26aab8a2e7d69758008793ae5605118acae2bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:01:32 -0700 Subject: [PATCH 074/105] fix --- tools/run_tests/python_utils/comment_on_pr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/run_tests/python_utils/comment_on_pr.py b/tools/run_tests/python_utils/comment_on_pr.py index 5c29abb0192..9708d0ffb49 100644 --- a/tools/run_tests/python_utils/comment_on_pr.py +++ b/tools/run_tests/python_utils/comment_on_pr.py @@ -46,5 +46,4 @@ def comment_on_pr(text): 'Authorization': 'token %s' % os.environ['JENKINS_OAUTH_TOKEN'], 'Content-Type': 'application/json', }) - with urllib2.urlopen(req) as rsp: - print rsp.read() + print urllib2.urlopen(req).read() From 1e578f62270fa86a20527110de3b566d11af9f41 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:03:46 -0700 Subject: [PATCH 075/105] Formatting --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index fa4cbf252da..c794b137111 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -212,5 +212,5 @@ if rows: text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') else: text = 'No significant performance differences' -comment_on_pr.comment_on_pr(text) +comment_on_pr.comment_on_pr('```%s```' % text) print text From 21e6a80f0c0822f2c8b4ecbdc6104ae6fcafe353 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:43:10 -0700 Subject: [PATCH 076/105] Review feedback --- tools/jenkins/run_performance.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 87699a283e7..dee033a951a 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -37,7 +37,4 @@ BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_c # Enter the gRPC repo root cd $(dirname $0)/../.. -# todo(mattkwong): Change performance test to use microbenchmarking tools/profiling/microbenchmarks/bm_diff.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN - -tools/run_tests/run_performance_tests.py -l c++ node ruby csharp python --netperf --category smoketest From 240b20da06e717ffdb921330093d5c8cef9a0f21 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:46:34 -0700 Subject: [PATCH 077/105] Review feedback --- tools/line_count/collect-history.py | 3 +++ tools/line_count/collect-now.sh | 2 +- tools/line_count/summarize-history.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/line_count/collect-history.py b/tools/line_count/collect-history.py index 858a6e1d30c..4c1bf73b1e1 100755 --- a/tools/line_count/collect-history.py +++ b/tools/line_count/collect-history.py @@ -31,6 +31,9 @@ import subprocess import datetime +# this script is only of historical interest: it's the script that was used to +# bootstrap the dataset + def daterange(start, end): for n in range(int((end - start).days)): yield start + datetime.timedelta(n) diff --git a/tools/line_count/collect-now.sh b/tools/line_count/collect-now.sh index 2db26298dc4..44f4b4ed310 100644 --- a/tools/line_count/collect-now.sh +++ b/tools/line_count/collect-now.sh @@ -31,6 +31,6 @@ set -ex cloc --vcs=git --by-file --yaml --out=cloc.yaml . -tools/line_count/yaml2csv -i cloc.yaml -d `date +%Y-%m-%d` -o cloc.csv +tools/line_count/yaml2csv.py -i cloc.yaml -d `date +%Y-%m-%d` -o cloc.csv bq load line_counts.grpc cloc.csv diff --git a/tools/line_count/summarize-history.py b/tools/line_count/summarize-history.py index 756dc79a4dc..cb6d570f665 100755 --- a/tools/line_count/summarize-history.py +++ b/tools/line_count/summarize-history.py @@ -32,6 +32,9 @@ import subprocess import datetime +# this script is only of historical interest: it's the script that was used to +# bootstrap the dataset + def daterange(start, end): for n in range(int((end - start).days)): yield start + datetime.timedelta(n) From 7af47feffc368b93720b02e8d2cb0e28c326604e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:54:23 -0700 Subject: [PATCH 078/105] Add jenkins run script --- tools/jenkins/run_line_count.sh | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tools/jenkins/run_line_count.sh diff --git a/tools/jenkins/run_line_count.sh b/tools/jenkins/run_line_count.sh new file mode 100644 index 00000000000..ee5370a6625 --- /dev/null +++ b/tools/jenkins/run_line_count.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and runs performance smoke test. +set -ex + +tools/line_count/collect-now.sh From 4b0e9aad7d4d7e713bff6128e4487a4481715316 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:55:06 -0700 Subject: [PATCH 079/105] Review feedback --- tools/jenkins/comment_on_pr.sh | 45 ---------------------------------- 1 file changed, 45 deletions(-) delete mode 100755 tools/jenkins/comment_on_pr.sh diff --git a/tools/jenkins/comment_on_pr.sh b/tools/jenkins/comment_on_pr.sh deleted file mode 100755 index 85f33aa917a..00000000000 --- a/tools/jenkins/comment_on_pr.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# This script is invoked by Jenkins to comment $1 on pull requests -# when triggered by a build - -set -e - -if [ -z "$1" ] || [ -z $JENKINS_OAUTH_TOKEN ] || [ -z $ghprbPullId ]; then - echo "Insufficient arguments or environment variables provided." - exit 1 -fi - -# Format the comment message to JSON -COMMENT_MESSAGE="{\"body\":\"$1\"}" - -curl -k -H "Authorization: token $JENKINS_OAUTH_TOKEN" -H "Content-Type: application/json" \ - -d "$COMMENT_MESSAGE" https://api.github.com/repos/grpc/grpc/issues/$ghprbPullId/comments From 123724f0a155d904a1f2a405ddc4d22f3043e321 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 16:16:59 -0700 Subject: [PATCH 080/105] Add new benchmark --- tools/profiling/microbenchmarks/bm_json.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index ca0af414a3b..31bcfb6aed0 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -113,6 +113,10 @@ _BM_SPECS = { 'BM_TransportStreamRecv': { 'tpl': [], 'dyn': ['request_size'], + }, + 'BM_StreamingPingPongWithCoalescingApi': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'request_count', 'end_of_stream'], } } From 31b26604087855a4eccaed643cc4a4380ffa7275 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 16:22:20 -0700 Subject: [PATCH 081/105] newlines --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index c794b137111..436d07bbf10 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -212,5 +212,5 @@ if rows: text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') else: text = 'No significant performance differences' -comment_on_pr.comment_on_pr('```%s```' % text) +comment_on_pr.comment_on_pr('```\n%s\n```' % text) print text From c2a52e36d87f5aaf96b9522768054788669dc6f7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 16:47:03 -0700 Subject: [PATCH 082/105] Tighten constraints --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index fa4cbf252da..ae0438cde57 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -83,8 +83,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=5) -argp.add_argument('-p', '--p_threshold', type=float, default=0.05) +argp.add_argument('-r', '--repetitions', type=int, default=10) +argp.add_argument('-p', '--p_threshold', type=float, default=0.01) args = argp.parse_args() assert args.diff_base From fc099cda09a40c8cbb02909ba3ccf6f0ec010362 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 17:13:31 -0700 Subject: [PATCH 083/105] Review feedback --- tools/jenkins/run_line_count.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/jenkins/run_line_count.sh b/tools/jenkins/run_line_count.sh index ee5370a6625..a2bde534130 100644 --- a/tools/jenkins/run_line_count.sh +++ b/tools/jenkins/run_line_count.sh @@ -28,7 +28,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# This script is invoked by Jenkins and runs performance smoke test. +# This script is invoked by Jenkins and counts the number of lines in the +# project. set -ex +cd $(dirname $0)/../.. tools/line_count/collect-now.sh From 07429fa9a6e0e705b8e297bd04ff41786b4f7f7a Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 29 Mar 2017 10:57:31 -0700 Subject: [PATCH 084/105] Update time_cmp to ignore tv_nsec when tv_sec is INT64 MAX or MIN --- src/core/lib/support/time.c | 2 +- test/core/support/time_test.c | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/core/lib/support/time.c b/src/core/lib/support/time.c index 614a068ddd3..c5f94d46f7b 100644 --- a/src/core/lib/support/time.c +++ b/src/core/lib/support/time.c @@ -42,7 +42,7 @@ int gpr_time_cmp(gpr_timespec a, gpr_timespec b) { int cmp = (a.tv_sec > b.tv_sec) - (a.tv_sec < b.tv_sec); GPR_ASSERT(a.clock_type == b.clock_type); - if (cmp == 0) { + if (cmp == 0 && a.tv_sec != INT64_MAX && a.tv_sec != INT64_MIN) { cmp = (a.tv_nsec > b.tv_nsec) - (a.tv_nsec < b.tv_nsec); } return cmp; diff --git a/test/core/support/time_test.c b/test/core/support/time_test.c index 73e43caa696..4cb36a788c1 100644 --- a/test/core/support/time_test.c +++ b/test/core/support/time_test.c @@ -256,11 +256,19 @@ static void test_similar(void) { } static void test_convert_extreme(void) { - gpr_timespec realtime_t = {INT64_MAX, 1, GPR_CLOCK_REALTIME}; - gpr_timespec monotime_t = - gpr_convert_clock_type(realtime_t, GPR_CLOCK_MONOTONIC); - GPR_ASSERT(monotime_t.tv_sec == realtime_t.tv_sec); - GPR_ASSERT(monotime_t.clock_type == GPR_CLOCK_MONOTONIC); + gpr_timespec realtime = {INT64_MAX, 1, GPR_CLOCK_REALTIME}; + gpr_timespec monotime = gpr_convert_clock_type(realtime, GPR_CLOCK_MONOTONIC); + GPR_ASSERT(monotime.tv_sec == realtime.tv_sec); + GPR_ASSERT(monotime.clock_type == GPR_CLOCK_MONOTONIC); +} + +static void test_cmp_extreme(void) { + gpr_timespec t1 = {INT64_MAX, 1, GPR_CLOCK_REALTIME}; + gpr_timespec t2 = {INT64_MAX, 2, GPR_CLOCK_REALTIME}; + GPR_ASSERT(gpr_time_cmp(t1, t2) == 0); + t1.tv_sec = INT64_MIN; + t2.tv_sec = INT64_MIN; + GPR_ASSERT(gpr_time_cmp(t1, t2) == 0); } int main(int argc, char *argv[]) { @@ -272,5 +280,6 @@ int main(int argc, char *argv[]) { test_sticky_infinities(); test_similar(); test_convert_extreme(); + test_cmp_extreme(); return 0; } From c410547eb889e79491fd91fc8d15fbc0669e95bf Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 29 Mar 2017 17:53:44 -0700 Subject: [PATCH 085/105] Give executable permission to line count .sh files --- tools/jenkins/run_line_count.sh | 0 tools/line_count/collect-now.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/jenkins/run_line_count.sh mode change 100644 => 100755 tools/line_count/collect-now.sh diff --git a/tools/jenkins/run_line_count.sh b/tools/jenkins/run_line_count.sh old mode 100644 new mode 100755 diff --git a/tools/line_count/collect-now.sh b/tools/line_count/collect-now.sh old mode 100644 new mode 100755 From a4ddf2d1d5341b73bf51352b1d14f6e549281fd6 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 29 Mar 2017 18:13:10 -0700 Subject: [PATCH 086/105] fix docker -p flag for interop tests --- tools/run_tests/run_interop_tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 1a83c10a42c..2d7f4a625d6 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -728,8 +728,7 @@ def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): ] else: - portstr = str(_DEFAULT_SERVER_PORT) - docker_args += ['-p', '%s:%s'%(portstr, portstr)] + docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, From 9b4a3cd984ee79478dcdb0d7bde757d78cb9db69 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 20:09:37 -0700 Subject: [PATCH 087/105] Reduce repetitions: this started taking too long --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 65fdc970a0a..4539ef7e418 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -83,7 +83,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=10) +argp.add_argument('-r', '--repetitions', type=int, default=7) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) args = argp.parse_args() From 76a3943ab9ff2926990fc43921b60d7074161a6a Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 29 Mar 2017 21:17:30 -0700 Subject: [PATCH 088/105] update dockerfile to version 2.6.1 of h2 --- .../interoptest/grpc_interop_http2/Dockerfile.template | 2 +- tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index a49e9512892..b517921f087 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -33,7 +33,7 @@ <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> - RUN pip install twisted h2 hyper + RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index da001528254..094a4e096df 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -47,7 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 -RUN pip install twisted h2 hyper +RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] From 23e6a8a67872c97f965345d747a689868b3cb68f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 07:24:01 -0700 Subject: [PATCH 089/105] Use medians, increase threshold --- tools/profiling/microbenchmarks/bm_diff.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4539ef7e418..2c4e572d12d 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -49,6 +49,14 @@ def changed_ratio(n, o): if o == 0: return 100 return (float(n)-float(o))/float(o) +def median(ary): + ary = sorted(ary) + n = len(ary) + if n%2 == 0: + return (ary[n/2] + ary[n/2+1]) / 2.0 + else: + return ary[n/2] + def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 @@ -90,8 +98,8 @@ args = argp.parse_args() assert args.diff_base def avg(lst): - sum = 0 - n = 0 + sum = 0.0 + n = 0.0 for el in lst: sum += el n += 1 @@ -162,11 +170,11 @@ class Benchmark: old = self.samples[False][f] if not new or not old: continue p = stats.ttest_ind(new, old)[1] - new_avg = avg(new) - old_avg = avg(old) - delta = new_avg - old_avg - ratio = changed_ratio(new_avg, old_avg) - if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.05: + new_mdn = median(new) + old_mdn = median(old) + delta = new_mdn - old_mdn + ratio = changed_ratio(new_mdn, old_mdn) + if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.1: self.final[f] = delta return self.final.keys() From 852b7d9fdef9472ccef14c8ba83fe62ceb2246b2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 08:15:16 -0700 Subject: [PATCH 090/105] Add debug: somethings going wrong --- tools/profiling/microbenchmarks/bm_diff.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 2c4e572d12d..f1d29ba3c64 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -174,7 +174,10 @@ class Benchmark: old_mdn = median(old) delta = new_mdn - old_mdn ratio = changed_ratio(new_mdn, old_mdn) - if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.1: + print 'new=%r old=%r new_mdn=%f old_mdn=%f delta=%f ratio=%f p=%f' % ( + new, old, new_mdn, old_mdn, delta, ratio, p + ) + if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.03: self.final[f] = delta return self.final.keys() @@ -207,7 +210,8 @@ for bm in comparables: benchmarks[name].add_sample(row, False) really_interesting = set() -for bm in benchmarks.values(): +for name, bm in benchmarks.items(): + print name really_interesting.update(bm.process()) fields = [f for f in _INTERESTING if f in really_interesting] From c212e6cc5e005526a80320ac95424e46dd341ad5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 08:26:21 -0700 Subject: [PATCH 091/105] Fix barrier, comment --- src/core/lib/iomgr/ev_epoll_linux.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 97f8f82fed9..f6372c0f3f6 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1173,9 +1173,9 @@ static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, switch (curr) { case CLOSURE_READY: case CLOSURE_NOT_READY: - /* Release cas to pair with a set_ready performing a load of the - shutdown state later */ - if (gpr_atm_rel_cas(state, curr, new_state)) { + /* Need a full barrier here so that the initial load in notify_on + doesn't need a barrier */ + if (gpr_atm_full_cas(state, curr, new_state)) { return; /* early out */ } break; /* retry */ From 05222512d7d8e3fe544b24468406be085388f8a6 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 21 Feb 2017 23:52:29 -0800 Subject: [PATCH 092/105] Add C++ large message tests to PRs and master --- tools/run_tests/generated/tests.json | 244 ++++++++++++++++++ .../run_tests/performance/scenario_config.py | 23 ++ 2 files changed, 267 insertions(+) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ca5d7a7a33a..8c4f82974bc 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -39573,6 +39573,56 @@ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure", "timeout_seconds": 360 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_secure", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure", + "timeout_seconds": 360 + }, { "args": [ "--scenarios_json", @@ -39902,6 +39952,56 @@ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure", "timeout_seconds": 360 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "tsan", + "asan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure", + "timeout_seconds": 360 + }, { "args": [ "--scenarios_json", @@ -40253,6 +40353,78 @@ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure_low_thread_count", "timeout_seconds": 360 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_secure_low_thread_count", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_secure_low_thread_count", + "timeout_seconds": 360 + }, { "args": [ "--scenarios_json", @@ -40725,6 +40897,78 @@ "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure_low_thread_count", "timeout_seconds": 360 }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_1channel_1MBmsg_insecure_low_thread_count", + "timeout_seconds": 360 + }, + { + "args": [ + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"num_clients\": 0, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" + ], + "boringssl": true, + "ci_platforms": [ + "linux" + ], + "cpu_cost": "capacity", + "defaults": "boringssl", + "exclude_configs": [ + "asan-noleaks", + "asan-trace-cmp", + "basicprof", + "counters", + "dbg", + "gcov", + "helgrind", + "memcheck", + "msan", + "mutrace", + "opt", + "stapprof", + "ubsan" + ], + "excluded_poll_engines": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux" + ], + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_64KBmsg_insecure_low_thread_count", + "timeout_seconds": 360 + }, { "args": [ "--scenarios_json", diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 1d91b61ba45..1870a738cac 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -214,6 +214,29 @@ class CXXLanguage: secure=secure, categories=smoketest_categories+[SCALABLE]) + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, + rpc_type='STREAMING', + req_size=1024*1024, + resp_size=1024*1024, + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, + secure=secure, + categories=smoketest_categories+[SCALABLE], + channels=1, outstanding=100) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % secstr, + rpc_type='STREAMING', + req_size=64*1024, + resp_size=64*1024, + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, + secure=secure, + categories=smoketest_categories+[SCALABLE]) + yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', From 55c895a6a426e7debba8589194095668cd3d2354 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 30 Mar 2017 10:43:14 -0700 Subject: [PATCH 093/105] Removed wrong LR include from codegen --- include/grpc++/impl/codegen/server_context.h | 1 - src/cpp/server/server_context.cc | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpc++/impl/codegen/server_context.h b/include/grpc++/impl/codegen/server_context.h index bf9a9b6f1a5..91f0be06e72 100644 --- a/include/grpc++/impl/codegen/server_context.h +++ b/include/grpc++/impl/codegen/server_context.h @@ -39,7 +39,6 @@ #include #include -#include #include #include diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index 05c05c86953..3a408eb23ed 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include #include From b88399932ef1966c4c0c7bf06c99012f389144ec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 10:54:14 -0700 Subject: [PATCH 094/105] Add new benchmark --- tools/profiling/microbenchmarks/bm_json.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 31bcfb6aed0..f9139431a01 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -117,6 +117,10 @@ _BM_SPECS = { 'BM_StreamingPingPongWithCoalescingApi': { 'tpl': ['fixture', 'client_mutator', 'server_mutator'], 'dyn': ['request_size', 'request_count', 'end_of_stream'], + }, + 'BM_Base16SomeStuff': { + 'tpl': [], + 'dyn': ['request_size'], } } From af5004653fcb568e8ebe8f0818a4524ca93f7581 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Thu, 30 Mar 2017 12:18:23 -0700 Subject: [PATCH 095/105] Add instructions to install libgflags-dev,libgtest-dev, clang and libc++-dev --- INSTALL.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 89f2f11f9da..ac03a5e6ac8 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -22,6 +22,12 @@ refer to these documents $ [sudo] apt-get install build-essential autoconf libtool ``` +If you plan to build from source and run tests, install the following as well: +```sh + $ [sudo] apt-get install libgflags-dev libgtest-dev + $ [sudo] apt-get install clang libc++-dev +``` + ## Mac OSX For a Mac system, git is not available by default. You will first need to From 04e5f65e2ad9245c803b3d3343945bc385742af3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 12:47:08 -0700 Subject: [PATCH 096/105] Add debug --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index f1d29ba3c64..0f54dbfe32e 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -125,6 +125,7 @@ def collect1(bm, cfg, ver): '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] + print cmd subprocess.check_call(cmd) build() From 9a212df417c26efa1d197a234f61c0c451bba8a1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 13:19:45 -0700 Subject: [PATCH 097/105] Add debug --- tools/profiling/microbenchmarks/bm_diff.py | 37 ++++++++++++++-------- tools/profiling/microbenchmarks/bm_json.py | 2 +- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 0f54dbfe32e..392c48ffe2d 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -60,14 +60,23 @@ def median(ary): def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 +nanos = { + 'abs_diff': 10, + 'pct_diff': 10, +} +counter = { + 'abs_diff': 1, + 'pct_diff': 1, +} + _INTERESTING = [ - 'cpu_time', - 'real_time', - 'locks_per_iteration', - 'allocs_per_iteration', - 'writes_per_iteration', - 'atm_cas_per_iteration', - 'atm_add_per_iteration', + 'cpu_time': nanos, + 'real_time': nanos, + 'locks_per_iteration': counter, + 'allocs_per_iteration': counter, + 'writes_per_iteration': counter, + 'atm_cas_per_iteration': counter, + 'atm_add_per_iteration': counter, ] _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', @@ -85,9 +94,9 @@ _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', - choices=sorted(_INTERESTING), + choices=sorted(_INTERESTING.keys()), nargs='+', - default=sorted(_INTERESTING), + default=sorted(_INTERESTING.keys()), help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) @@ -161,12 +170,12 @@ class Benchmark: self.final = {} def add_sample(self, data, new): - for f in _INTERESTING: + for f in args.track: if f in data: self.samples[new][f].append(float(data[f])) def process(self): - for f in _INTERESTING: + for f in args.track: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue @@ -178,7 +187,7 @@ class Benchmark: print 'new=%r old=%r new_mdn=%f old_mdn=%f delta=%f ratio=%f p=%f' % ( new, old, new_mdn, old_mdn, delta, ratio, p ) - if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.03: + if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']: self.final[f] = delta return self.final.keys() @@ -202,10 +211,12 @@ for bm in comparables: js_old_opt = json.loads(f.read()) for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, True) for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, False) @@ -214,7 +225,7 @@ really_interesting = set() for name, bm in benchmarks.items(): print name really_interesting.update(bm.process()) -fields = [f for f in _INTERESTING if f in really_interesting] +fields = [f for f in args.track if f in args.track] headers = ['Benchmark'] + fields rows = [] diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 1ec441efee4..1a618a690d0 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -179,7 +179,7 @@ def parse_name(name): def expand_json(js, js2 = None): for bm in js['benchmarks']: - if bm['name'].endswith('_stddev') or bm['name'].endswith('mean'): continue + if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue context = js['context'] if 'label' in bm: labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] From d2461e522a32d4fb97efa39b48a18a67603527c7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 13:20:17 -0700 Subject: [PATCH 098/105] Add debug --- tools/profiling/microbenchmarks/bm_diff.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 392c48ffe2d..192338ca8e0 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -69,7 +69,7 @@ counter = { 'pct_diff': 1, } -_INTERESTING = [ +_INTERESTING = { 'cpu_time': nanos, 'real_time': nanos, 'locks_per_iteration': counter, @@ -77,7 +77,8 @@ _INTERESTING = [ 'writes_per_iteration': counter, 'atm_cas_per_iteration': counter, 'atm_add_per_iteration': counter, -] +} + _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', From 03bab0af2ebdd0e690e2362d8d2c4d06d58f9629 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 13:24:59 -0700 Subject: [PATCH 099/105] Fix merge error --- tools/profiling/microbenchmarks/bm_json.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 1a618a690d0..cd39d707a64 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -177,6 +177,8 @@ def parse_name(name): out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) return out +used_poison = {} + def expand_json(js, js2 = None): for bm in js['benchmarks']: if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue @@ -198,8 +200,9 @@ def expand_json(js, js2 = None): row.update(labels) if js2: for bm2 in js2['benchmarks']: - if bm['name'] == bm2['name']: + if bm['name'] == bm2['name'] and used_poison not in bm2: row['cpu_time'] = bm2['cpu_time'] row['real_time'] = bm2['real_time'] row['iterations'] = bm2['iterations'] + bm2[used_poison] = True yield row From c78c9158c085245f9fee3b8307fa7434ea2fdd7f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 13:29:02 -0700 Subject: [PATCH 100/105] Fix merge error --- tools/profiling/microbenchmarks/bm_json.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index cd39d707a64..e885444f410 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -177,8 +177,6 @@ def parse_name(name): out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) return out -used_poison = {} - def expand_json(js, js2 = None): for bm in js['benchmarks']: if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue @@ -200,9 +198,9 @@ def expand_json(js, js2 = None): row.update(labels) if js2: for bm2 in js2['benchmarks']: - if bm['name'] == bm2['name'] and used_poison not in bm2: + if bm['name'] == bm2['name'] and 'already_used' not in bm2: row['cpu_time'] = bm2['cpu_time'] row['real_time'] = bm2['real_time'] row['iterations'] = bm2['iterations'] - bm2[used_poison] = True + bm2['already_used'] = True yield row From 1463d0e74df002e8d48515ad03d132f5f4cf7ad2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 13:40:57 -0700 Subject: [PATCH 101/105] Spam cleanup --- tools/profiling/microbenchmarks/bm_diff.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 192338ca8e0..857d397095b 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -185,9 +185,6 @@ class Benchmark: old_mdn = median(old) delta = new_mdn - old_mdn ratio = changed_ratio(new_mdn, old_mdn) - print 'new=%r old=%r new_mdn=%f old_mdn=%f delta=%f ratio=%f p=%f' % ( - new, old, new_mdn, old_mdn, delta, ratio, p - ) if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']: self.final[f] = delta return self.final.keys() @@ -212,19 +209,16 @@ for bm in comparables: js_old_opt = json.loads(f.read()) for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, True) for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, False) really_interesting = set() for name, bm in benchmarks.items(): - print name really_interesting.update(bm.process()) fields = [f for f in args.track if f in args.track] From ee8c107d26f12402ea56a0117e3134c0f1d1f0b5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 16:58:46 -0700 Subject: [PATCH 102/105] xyz --- build.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.yaml b/build.yaml index 974ae9cc6bd..4b9a6f07030 100644 --- a/build.yaml +++ b/build.yaml @@ -4311,6 +4311,9 @@ configs: opt: CPPFLAGS: -O2 DEFINES: NDEBUG + lto: + CPPFLAGS: -O2 + DEFINES: NDEBUG stapprof: CPPFLAGS: -O2 -DGRPC_STAP_PROFILER DEFINES: NDEBUG From d4387c007507d57ccbd849cf62691d9b959152c9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 17:00:43 -0700 Subject: [PATCH 103/105] codgen --- Makefile | 114 +++++++++++++------------ build.yaml | 6 +- tools/run_tests/generated/configs.json | 51 +++++------ tools/run_tests/generated/tests.json | 30 +++++++ 4 files changed, 121 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index 5100c7fda01..a28a27c0dae 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,42 @@ LDXX_opt = $(DEFAULT_CXX) CPPFLAGS_opt = -O2 DEFINES_opt = NDEBUG +VALID_CONFIG_asan-trace-cmp = 1 +REQUIRE_CUSTOM_LIBRARIES_asan-trace-cmp = 1 +CC_asan-trace-cmp = clang +CXX_asan-trace-cmp = clang++ +LD_asan-trace-cmp = clang +LDXX_asan-trace-cmp = clang++ +CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +LDFLAGS_asan-trace-cmp = -fsanitize=address + +VALID_CONFIG_dbg = 1 +CC_dbg = $(DEFAULT_CC) +CXX_dbg = $(DEFAULT_CXX) +LD_dbg = $(DEFAULT_CC) +LDXX_dbg = $(DEFAULT_CXX) +CPPFLAGS_dbg = -O0 +DEFINES_dbg = _DEBUG DEBUG + +VALID_CONFIG_asan = 1 +REQUIRE_CUSTOM_LIBRARIES_asan = 1 +CC_asan = clang +CXX_asan = clang++ +LD_asan = clang +LDXX_asan = clang++ +CPPFLAGS_asan = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +LDFLAGS_asan = -fsanitize=address + +VALID_CONFIG_msan = 1 +REQUIRE_CUSTOM_LIBRARIES_msan = 1 +CC_msan = clang +CXX_msan = clang++ +LD_msan = clang +LDXX_msan = clang++ +CPPFLAGS_msan = -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS +LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) +DEFINES_msan = NDEBUG + VALID_CONFIG_basicprof = 1 CC_basicprof = $(DEFAULT_CC) CXX_basicprof = $(DEFAULT_CXX) @@ -121,22 +157,25 @@ LDXX_asan-noleaks = clang++ CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address -VALID_CONFIG_asan-trace-cmp = 1 -REQUIRE_CUSTOM_LIBRARIES_asan-trace-cmp = 1 -CC_asan-trace-cmp = clang -CXX_asan-trace-cmp = clang++ -LD_asan-trace-cmp = clang -LDXX_asan-trace-cmp = clang++ -CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS -LDFLAGS_asan-trace-cmp = -fsanitize=address +VALID_CONFIG_ubsan = 1 +REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 +CC_ubsan = clang +CXX_ubsan = clang++ +LD_ubsan = clang +LDXX_ubsan = clang++ +CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs +LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow +DEFINES_ubsan = NDEBUG -VALID_CONFIG_dbg = 1 -CC_dbg = $(DEFAULT_CC) -CXX_dbg = $(DEFAULT_CXX) -LD_dbg = $(DEFAULT_CC) -LDXX_dbg = $(DEFAULT_CXX) -CPPFLAGS_dbg = -O0 -DEFINES_dbg = _DEBUG DEBUG +VALID_CONFIG_tsan = 1 +REQUIRE_CUSTOM_LIBRARIES_tsan = 1 +CC_tsan = clang +CXX_tsan = clang++ +LD_tsan = clang +LDXX_tsan = clang++ +CPPFLAGS_tsan = -O0 -fsanitize=thread -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS +LDFLAGS_tsan = -fsanitize=thread +DEFINES_tsan = GRPC_TSAN VALID_CONFIG_stapprof = 1 CC_stapprof = $(DEFAULT_CC) @@ -164,44 +203,13 @@ CPPFLAGS_memcheck = -O0 LDFLAGS_memcheck = -rdynamic DEFINES_memcheck = _DEBUG DEBUG -VALID_CONFIG_asan = 1 -REQUIRE_CUSTOM_LIBRARIES_asan = 1 -CC_asan = clang -CXX_asan = clang++ -LD_asan = clang -LDXX_asan = clang++ -CPPFLAGS_asan = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS -LDFLAGS_asan = -fsanitize=address - -VALID_CONFIG_tsan = 1 -REQUIRE_CUSTOM_LIBRARIES_tsan = 1 -CC_tsan = clang -CXX_tsan = clang++ -LD_tsan = clang -LDXX_tsan = clang++ -CPPFLAGS_tsan = -O0 -fsanitize=thread -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS -LDFLAGS_tsan = -fsanitize=thread -DEFINES_tsan = GRPC_TSAN - -VALID_CONFIG_ubsan = 1 -REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 -CC_ubsan = clang -CXX_ubsan = clang++ -LD_ubsan = clang -LDXX_ubsan = clang++ -CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs -LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow -DEFINES_ubsan = NDEBUG - -VALID_CONFIG_msan = 1 -REQUIRE_CUSTOM_LIBRARIES_msan = 1 -CC_msan = clang -CXX_msan = clang++ -LD_msan = clang -LDXX_msan = clang++ -CPPFLAGS_msan = -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS -LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) -DEFINES_msan = NDEBUG +VALID_CONFIG_lto = 1 +CC_lto = $(DEFAULT_CC) +CXX_lto = $(DEFAULT_CXX) +LD_lto = $(DEFAULT_CC) +LDXX_lto = $(DEFAULT_CXX) +CPPFLAGS_lto = -O2 +DEFINES_lto = NDEBUG VALID_CONFIG_mutrace = 1 CC_mutrace = $(DEFAULT_CC) diff --git a/build.yaml b/build.yaml index 4b9a6f07030..0a0b84dfb61 100644 --- a/build.yaml +++ b/build.yaml @@ -4287,6 +4287,9 @@ configs: DEFINES: _DEBUG DEBUG LDFLAGS: -rdynamic valgrind: --tool=helgrind + lto: + CPPFLAGS: -O2 + DEFINES: NDEBUG memcheck: CPPFLAGS: -O0 DEFINES: _DEBUG DEBUG @@ -4311,9 +4314,6 @@ configs: opt: CPPFLAGS: -O2 DEFINES: NDEBUG - lto: - CPPFLAGS: -O2 - DEFINES: NDEBUG stapprof: CPPFLAGS: -O2 -DGRPC_STAP_PROFILER DEFINES: NDEBUG diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index 69e0f447743..93dd6fb3d43 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -2,6 +2,26 @@ { "config": "opt" }, + { + "config": "asan-trace-cmp", + "environ": { + "ASAN_OPTIONS": "detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + } + }, + { + "config": "dbg" + }, + { + "config": "asan", + "environ": { + "ASAN_OPTIONS": "detect_leaks=1:color=always", + "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + } + }, + { + "config": "msan" + }, { "config": "basicprof" }, @@ -19,14 +39,16 @@ } }, { - "config": "asan-trace-cmp", + "config": "ubsan", "environ": { - "ASAN_OPTIONS": "detect_leaks=1:color=always", - "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" + "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt" } }, { - "config": "dbg" + "config": "tsan", + "environ": { + "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" + } }, { "config": "stapprof" @@ -43,26 +65,7 @@ ] }, { - "config": "asan", - "environ": { - "ASAN_OPTIONS": "detect_leaks=1:color=always", - "LSAN_OPTIONS": "suppressions=tools/lsan_suppressions.txt:report_objects=1" - } - }, - { - "config": "tsan", - "environ": { - "TSAN_OPTIONS": "suppressions=tools/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" - } - }, - { - "config": "ubsan", - "environ": { - "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1:suppressions=tools/ubsan_suppressions.txt" - } - }, - { - "config": "msan" + "config": "lto" }, { "config": "mutrace" diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 8c4f82974bc..43879ff9b96 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -40300,6 +40300,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40336,6 +40337,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40372,6 +40374,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40408,6 +40411,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40444,6 +40448,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40480,6 +40485,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40518,6 +40524,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40556,6 +40563,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40592,6 +40600,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40628,6 +40637,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40664,6 +40674,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40700,6 +40711,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40736,6 +40748,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40772,6 +40785,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40808,6 +40822,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40844,6 +40859,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40880,6 +40896,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40916,6 +40933,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40952,6 +40970,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -40988,6 +41007,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41024,6 +41044,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41062,6 +41083,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41100,6 +41122,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41136,6 +41159,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41172,6 +41196,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41208,6 +41233,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41244,6 +41270,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41280,6 +41307,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41316,6 +41344,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", @@ -41352,6 +41381,7 @@ "dbg", "gcov", "helgrind", + "lto", "memcheck", "msan", "mutrace", From b5946a7546a7ca00c2bae5ca28ae6f591b49a9a7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 20:36:44 -0700 Subject: [PATCH 104/105] Loosen constraints for reporting --- tools/profiling/microbenchmarks/bm_diff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 857d397095b..0e20a0e4e51 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -61,8 +61,8 @@ def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 nanos = { - 'abs_diff': 10, - 'pct_diff': 10, + 'abs_diff': 5, + 'pct_diff': 5, } counter = { 'abs_diff': 1, @@ -102,7 +102,7 @@ argp.add_argument('-t', '--track', argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=7) -argp.add_argument('-p', '--p_threshold', type=float, default=0.01) +argp.add_argument('-p', '--p_threshold', type=float, default=0.05) args = argp.parse_args() assert args.diff_base From a77023b3d26c1444f61b36ce567eeab72d46a95a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 20:42:42 -0700 Subject: [PATCH 105/105] Update bm_diff.py --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 0e20a0e4e51..339e9595185 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -101,7 +101,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=7) +argp.add_argument('-r', '--repetitions', type=int, default=4) argp.add_argument('-p', '--p_threshold', type=float, default=0.05) args = argp.parse_args()