From ea525ebd3949f89923f30da22471495e9e066326 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 17:50:32 +0000 Subject: [PATCH 1/8] Threaded port_server --- tools/run_tests/python_utils/port_server.py | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py index dbd32efc0e8..67ae4715820 100755 --- a/tools/run_tests/python_utils/port_server.py +++ b/tools/run_tests/python_utils/port_server.py @@ -33,18 +33,20 @@ from __future__ import print_function import argparse -from six.moves import BaseHTTPServer +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler import hashlib import os import socket import sys import time +from SocketServer import ThreadingMixIn +import threading # increment this number whenever making a change to ensure that # the changes are picked up by running CI servers # note that all changes must be backwards compatible -_MY_VERSION = 9 +_MY_VERSION = 10 if len(sys.argv) == 2 and sys.argv[1] == 'dump_version': @@ -111,12 +113,12 @@ def allocate_port(req): keep_running = True -class Handler(BaseHTTPServer.BaseHTTPRequestHandler): +class Handler(BaseHTTPRequestHandler): def setup(self): # If the client is unreachable for 5 seconds, close the connection self.timeout = 5 - BaseHTTPServer.BaseHTTPRequestHandler.setup(self) + BaseHTTPRequestHandler.setup(self) def do_GET(self): global keep_running @@ -158,12 +160,12 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler): elif self.path == '/quitquitquit': self.send_response(200) self.end_headers() - keep_running = False + sys.exit(0) +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + """Handle requests in a separate thread""" -httpd = BaseHTTPServer.HTTPServer(('', args.port), Handler) -while keep_running: - httpd.handle_request() - sys.stderr.flush() -print('done') +httpd = ThreadedHTTPServer(('', args.port), Handler) +httpd.serve_forever() + From 25149249ec74eade6af512f2573c694c34fab9fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 13:42:52 -0700 Subject: [PATCH 2/8] Slow down timers for ubsan builds --- Makefile | 2 +- build.yaml | 2 +- test/core/util/test_config.c | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a650655a7d5..58a36332d51 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ LD_ubsan = clang LDXX_ubsan = clang++ CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow -DEFINES_ubsan = NDEBUG +DEFINES_ubsan = NDEBUG GRPC_UBSAN VALID_CONFIG_tsan = 1 REQUIRE_CUSTOM_LIBRARIES_tsan = 1 diff --git a/build.yaml b/build.yaml index 58a474aabf1..5791402f575 100644 --- a/build.yaml +++ b/build.yaml @@ -4464,7 +4464,7 @@ configs: CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs CXX: clang++ - DEFINES: NDEBUG + DEFINES: NDEBUG GRPC_UBSAN LD: clang LDFLAGS: -fsanitize=undefined,unsigned-integer-overflow LDXX: clang++ diff --git a/test/core/util/test_config.c b/test/core/util/test_config.c index 0180d6f08d4..9a400c54ca2 100644 --- a/test/core/util/test_config.c +++ b/test/core/util/test_config.c @@ -348,6 +348,14 @@ bool BuiltUnderMsan() { #endif } +bool BuiltUnderUbsan() { +#ifdef GRPC_UBSAN + return true; +#else + return false; +#endif +} + int64_t grpc_test_sanitizer_slowdown_factor() { int64_t sanitizer_multiplier = 1; if (BuiltUnderValgrind()) { @@ -358,6 +366,8 @@ int64_t grpc_test_sanitizer_slowdown_factor() { sanitizer_multiplier = 3; } else if (BuiltUnderMsan()) { sanitizer_multiplier = 4; + } else if (BuiltUnderUbsan()) { + sanitizer_multiplier = 5; } return sanitizer_multiplier; } From ec4952426976c60e33fb279aa36a1cd1f89bd938 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 20:55:43 +0000 Subject: [PATCH 3/8] Add a mutex around allocate_port --- tools/run_tests/python_utils/port_server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py index 67ae4715820..522cbed9e1c 100755 --- a/tools/run_tests/python_utils/port_server.py +++ b/tools/run_tests/python_utils/port_server.py @@ -46,7 +46,7 @@ import threading # increment this number whenever making a change to ensure that # the changes are picked up by running CI servers # note that all changes must be backwards compatible -_MY_VERSION = 10 +_MY_VERSION = 11 if len(sys.argv) == 2 and sys.argv[1] == 'dump_version': @@ -70,6 +70,7 @@ print('port server running on port %d' % args.port) pool = [] in_use = {} +mu = threading.Lock() def refill_pool(max_timeout, req): @@ -97,16 +98,21 @@ def refill_pool(max_timeout, req): def allocate_port(req): global pool global in_use + global mu + mu.acquire() max_timeout = 600 while not pool: refill_pool(max_timeout, req) if not pool: req.log_message("failed to find ports: retrying soon") + mu.release() time.sleep(1) + mu.acquire() max_timeout /= 2 port = pool[0] pool = pool[1:] in_use[port] = time.time() + mu.release() return port From b56dae554ca14dfd5a76690462b9efbc38314de0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 08:22:24 -0700 Subject: [PATCH 4/8] Allow a longer fetch time for port server --- test/core/util/port_server_client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 38054dd1e74..254c3a6b613 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -103,7 +103,7 @@ void grpc_free_port_using_server(int port) { grpc_resource_quota *resource_quota = grpc_resource_quota_create("port_server_client/free"); grpc_httpcli_get(&exec_ctx, &context, &pr.pops, resource_quota, &req, - grpc_timeout_seconds_to_deadline(10), + grpc_timeout_seconds_to_deadline(30), grpc_closure_create(freed_port_from_server, &pr, grpc_schedule_on_exec_ctx), &rsp); @@ -235,7 +235,7 @@ int grpc_pick_port_using_server(void) { grpc_resource_quota_create("port_server_client/pick"); grpc_httpcli_get( &exec_ctx, &context, &pr.pops, resource_quota, &req, - grpc_timeout_seconds_to_deadline(10), + grpc_timeout_seconds_to_deadline(30), grpc_closure_create(got_port_from_server, &pr, grpc_schedule_on_exec_ctx), &pr.response); grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); From c9c6aa7df0442a2861619a6403380350420871c4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 08:27:31 -0700 Subject: [PATCH 5/8] Be resilient against failed runs --- tools/profiling/microbenchmarks/bm_diff.py | 27 +++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 6ee4cbfc7bb..3c15478774e 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -204,7 +204,10 @@ def eintr_be_gone(fn): def read_json(filename): - with open(filename) as f: return json.loads(f.read()) + try: + with open(filename) as f: return json.loads(f.read()) + except ValueError, e: + return None def finalize(): @@ -217,16 +220,18 @@ def finalize(): js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, True) - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, False) + if js_new_ctr: + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, True) + if js_old_ctr: + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, False) really_interesting = set() for name, bm in benchmarks.items(): From 9fd7cf5dc3fca5a07df03b2dda08519a2e6b308a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 08:34:14 -0700 Subject: [PATCH 6/8] Recycle ports --- test/cpp/microbenchmarks/fullstack_fixtures.h | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index f129ede26a6..98aca1c3465 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -113,13 +113,17 @@ class TCP : public FullstackFixture { public: TCP(Service* service, const FixtureConfiguration& fixture_configuration = FixtureConfiguration()) - : FullstackFixture(service, fixture_configuration, MakeAddress()) {} + : FullstackFixture(service, fixture_configuration, MakeAddress(&port_)) {} + + ~TCP() { grpc_recycle_unused_port(port_); } private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); + int port_; + + static grpc::string MakeAddress(int* port) { + *port = grpc_pick_unused_port_or_die(); std::stringstream addr; - addr << "localhost:" << port; + addr << "localhost:" << *port; return addr.str(); } }; @@ -128,14 +132,18 @@ class UDS : public FullstackFixture { public: UDS(Service* service, const FixtureConfiguration& fixture_configuration = FixtureConfiguration()) - : FullstackFixture(service, fixture_configuration, MakeAddress()) {} + : FullstackFixture(service, fixture_configuration, MakeAddress(&port_)) {} + + ~UDS() { grpc_recycle_unused_port(port_); } private: - static grpc::string MakeAddress() { - int port = grpc_pick_unused_port_or_die(); // just for a unique id - not a - // real port + int port_; + + static grpc::string MakeAddress(int* port) { + *port = grpc_pick_unused_port_or_die(); // just for a unique id - not a + // real port std::stringstream addr; - addr << "unix:/tmp/bm_fullstack." << port; + addr << "unix:/tmp/bm_fullstack." << *port; return addr.str(); } }; From a27edc69c3e9b957a32f8d89f4969abe6cc37580 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 09:20:30 -0700 Subject: [PATCH 7/8] Ensure port server running --- tools/jenkins/run_performance.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 544e31dcbdb..f530fb46b86 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -37,4 +37,5 @@ BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong # Enter the gRPC repo root cd $(dirname $0)/../.. +tools/run_tests/start_port_server.py tools/profiling/microbenchmarks/bm_diff.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN From 09ebed7bf4c19a8c31195b44c8d911f39779323e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 10:19:10 -0700 Subject: [PATCH 8/8] Ensure port server can shutdown --- tools/run_tests/python_utils/port_server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py index 522cbed9e1c..e96ee0b08c1 100755 --- a/tools/run_tests/python_utils/port_server.py +++ b/tools/run_tests/python_utils/port_server.py @@ -46,7 +46,7 @@ import threading # increment this number whenever making a change to ensure that # the changes are picked up by running CI servers # note that all changes must be backwards compatible -_MY_VERSION = 11 +_MY_VERSION = 14 if len(sys.argv) == 2 and sys.argv[1] == 'dump_version': @@ -166,12 +166,11 @@ class Handler(BaseHTTPRequestHandler): elif self.path == '/quitquitquit': self.send_response(200) self.end_headers() - sys.exit(0) + self.server.shutdown() class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): """Handle requests in a separate thread""" -httpd = ThreadedHTTPServer(('', args.port), Handler) -httpd.serve_forever() +ThreadedHTTPServer(('', args.port), Handler).serve_forever()