Merge branch 'master' into cq_mpsc_based

Makefile

@@ -174,7 +174,7 @@ LD_ubsan = clang
 LDXX_ubsan = clang++
 CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs
 LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow
-DEFINES_ubsan = NDEBUG
+DEFINES_ubsan = NDEBUG GRPC_UBSAN
 
 VALID_CONFIG_tsan = 1
 REQUIRE_CUSTOM_LIBRARIES_tsan = 1

build.yaml

@@ -4467,7 +4467,7 @@ configs:
     CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer
       -Wno-unused-command-line-argument -Wvarargs
     CXX: clang++
-    DEFINES: NDEBUG
+    DEFINES: NDEBUG GRPC_UBSAN
     LD: clang
     LDFLAGS: -fsanitize=undefined,unsigned-integer-overflow
     LDXX: clang++

test/core/util/port_server_client.c

@@ -103,7 +103,7 @@ void grpc_free_port_using_server(int port) {
   grpc_resource_quota *resource_quota =
       grpc_resource_quota_create("port_server_client/free");
   grpc_httpcli_get(&exec_ctx, &context, &pr.pops, resource_quota, &req,
-                   grpc_timeout_seconds_to_deadline(10),
+                   grpc_timeout_seconds_to_deadline(30),
                    grpc_closure_create(freed_port_from_server, &pr,
                                        grpc_schedule_on_exec_ctx),
                    &rsp);
@@ -235,7 +235,7 @@ int grpc_pick_port_using_server(void) {
       grpc_resource_quota_create("port_server_client/pick");
   grpc_httpcli_get(
       &exec_ctx, &context, &pr.pops, resource_quota, &req,
-      grpc_timeout_seconds_to_deadline(10),
+      grpc_timeout_seconds_to_deadline(30),
       grpc_closure_create(got_port_from_server, &pr, grpc_schedule_on_exec_ctx),
       &pr.response);
   grpc_resource_quota_unref_internal(&exec_ctx, resource_quota);

test/core/util/test_config.c

@@ -348,6 +348,14 @@ bool BuiltUnderMsan() {
 #endif
 }
 
+bool BuiltUnderUbsan() {
+#ifdef GRPC_UBSAN
+  return true;
+#else
+  return false;
+#endif
+}
+
 int64_t grpc_test_sanitizer_slowdown_factor() {
   int64_t sanitizer_multiplier = 1;
   if (BuiltUnderValgrind()) {
@@ -358,6 +366,8 @@ int64_t grpc_test_sanitizer_slowdown_factor() {
     sanitizer_multiplier = 3;
   } else if (BuiltUnderMsan()) {
     sanitizer_multiplier = 4;
+  } else if (BuiltUnderUbsan()) {
+    sanitizer_multiplier = 5;
   }
   return sanitizer_multiplier;
 }

test/cpp/microbenchmarks/fullstack_fixtures.h

@@ -113,13 +113,17 @@ class TCP : public FullstackFixture {
  public:
   TCP(Service* service, const FixtureConfiguration& fixture_configuration =
                             FixtureConfiguration())
-      : FullstackFixture(service, fixture_configuration, MakeAddress()) {}
+      : FullstackFixture(service, fixture_configuration, MakeAddress(&port_)) {}
+
+  ~TCP() { grpc_recycle_unused_port(port_); }
 
  private:
-  static grpc::string MakeAddress() {
-    int port = grpc_pick_unused_port_or_die();
+  int port_;
+
+  static grpc::string MakeAddress(int* port) {
+    *port = grpc_pick_unused_port_or_die();
     std::stringstream addr;
-    addr << "localhost:" << port;
+    addr << "localhost:" << *port;
     return addr.str();
   }
 };
@@ -128,14 +132,18 @@ class UDS : public FullstackFixture {
  public:
   UDS(Service* service, const FixtureConfiguration& fixture_configuration =
                             FixtureConfiguration())
-      : FullstackFixture(service, fixture_configuration, MakeAddress()) {}
+      : FullstackFixture(service, fixture_configuration, MakeAddress(&port_)) {}
+
+  ~UDS() { grpc_recycle_unused_port(port_); }
 
  private:
-  static grpc::string MakeAddress() {
-    int port = grpc_pick_unused_port_or_die();  // just for a unique id - not a
-                                                // real port
+  int port_;
+
+  static grpc::string MakeAddress(int* port) {
+    *port = grpc_pick_unused_port_or_die();  // just for a unique id - not a
+                                             // real port
     std::stringstream addr;
-    addr << "unix:/tmp/bm_fullstack." << port;
+    addr << "unix:/tmp/bm_fullstack." << *port;
     return addr.str();
   }
 };

tools/jenkins/run_performance.sh

@@ -37,4 +37,5 @@ BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong
 # Enter the gRPC repo root
 cd $(dirname $0)/../..
 
+tools/run_tests/start_port_server.py
 tools/profiling/microbenchmarks/bm_diff.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN

tools/profiling/microbenchmarks/bm_diff.py

@@ -204,7 +204,10 @@ def eintr_be_gone(fn):
 
 
 def read_json(filename):
-  with open(filename) as f: return json.loads(f.read())
+  try:
+    with open(filename) as f: return json.loads(f.read())
+  except ValueError, e:
+    return None
 
 
 def finalize():
@@ -217,16 +220,18 @@ def finalize():
       js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop))
       js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop))
 
-      for row in bm_json.expand_json(js_new_ctr, js_new_opt):
-        print row
-        name = row['cpp_name']
-        if name.endswith('_mean') or name.endswith('_stddev'): continue
-        benchmarks[name].add_sample(row, True)
-      for row in bm_json.expand_json(js_old_ctr, js_old_opt):
-        print row
-        name = row['cpp_name']
-        if name.endswith('_mean') or name.endswith('_stddev'): continue
-        benchmarks[name].add_sample(row, False)
+      if js_new_ctr:
+        for row in bm_json.expand_json(js_new_ctr, js_new_opt):
+          print row
+          name = row['cpp_name']
+          if name.endswith('_mean') or name.endswith('_stddev'): continue
+          benchmarks[name].add_sample(row, True)
+      if js_old_ctr:
+        for row in bm_json.expand_json(js_old_ctr, js_old_opt):
+          print row
+          name = row['cpp_name']
+          if name.endswith('_mean') or name.endswith('_stddev'): continue
+          benchmarks[name].add_sample(row, False)
 
   really_interesting = set()
   for name, bm in benchmarks.items():

tools/run_tests/python_utils/port_server.py

@@ -33,18 +33,20 @@
 from __future__ import print_function
 
 import argparse
-from six.moves import BaseHTTPServer
+from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
 import hashlib
 import os
 import socket
 import sys
 import time
+from SocketServer import ThreadingMixIn
+import threading
 
 
 # increment this number whenever making a change to ensure that
 # the changes are picked up by running CI servers
 # note that all changes must be backwards compatible
-_MY_VERSION = 9
+_MY_VERSION = 14
 
 
 if len(sys.argv) == 2 and sys.argv[1] == 'dump_version':
@@ -68,6 +70,7 @@ print('port server running on port %d' % args.port)
 
 pool = []
 in_use = {}
+mu = threading.Lock()
 
 
 def refill_pool(max_timeout, req):
@@ -95,28 +98,33 @@ def refill_pool(max_timeout, req):
 def allocate_port(req):
   global pool
   global in_use
+  global mu
+  mu.acquire()
   max_timeout = 600
   while not pool:
     refill_pool(max_timeout, req)
     if not pool:
       req.log_message("failed to find ports: retrying soon")
+      mu.release()
       time.sleep(1)
+      mu.acquire()
       max_timeout /= 2
   port = pool[0]
   pool = pool[1:]
   in_use[port] = time.time()
+  mu.release()
   return port
 
 
 keep_running = True
 
 
-class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
+class Handler(BaseHTTPRequestHandler):
   
   def setup(self):
     # If the client is unreachable for 5 seconds, close the connection
     self.timeout = 5
-    BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
+    BaseHTTPRequestHandler.setup(self)
 
   def do_GET(self):
     global keep_running
@@ -158,12 +166,11 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
     elif self.path == '/quitquitquit':
       self.send_response(200)
       self.end_headers()
-      keep_running = False
+      self.server.shutdown()
 
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+  """Handle requests in a separate thread"""
 
-httpd = BaseHTTPServer.HTTPServer(('', args.port), Handler)
-while keep_running:
-  httpd.handle_request()
-  sys.stderr.flush()
 
-print('done')
+ThreadedHTTPServer(('', args.port), Handler).serve_forever()
+