Merge branch 'test_affine' into reuse_affinity

build.yaml

@@ -3277,7 +3277,7 @@ configs:
     LDXX: clang++
     compile_the_world: true
     test_environ:
-      UBSAN_OPTIONS: print_stacktrace=1
+      UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
     timeout_multiplier: 1.5
 defaults:
   boringssl:

src/core/ext/client_config/subchannel.c

@@ -320,7 +320,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
     c->filters = NULL;
   }
   c->addr = gpr_malloc(args->addr_len);
-  memcpy(c->addr, args->addr, args->addr_len);
+  if (args->addr_len) memcpy(c->addr, args->addr, args->addr_len);
   c->pollset_set = grpc_pollset_set_create();
   c->addr_len = args->addr_len;
   grpc_set_initial_connect_string(&c->addr, &c->addr_len,

src/core/ext/client_config/subchannel_index.c

@@ -77,12 +77,19 @@ static grpc_subchannel_key *create_key(
   grpc_subchannel_key *k = gpr_malloc(sizeof(*k));
   k->connector = grpc_connector_ref(connector);
   k->args.filter_count = args->filter_count;
-  k->args.filters = gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
+  if (k->args.filter_count > 0) {
-  memcpy((grpc_channel_filter *)k->args.filters, args->filters,
+    k->args.filters =
-         sizeof(*k->args.filters) * k->args.filter_count);
+        gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
+    memcpy((grpc_channel_filter *)k->args.filters, args->filters,
+           sizeof(*k->args.filters) * k->args.filter_count);
+  } else {
+    k->args.filters = NULL;
+  }
   k->args.addr_len = args->addr_len;
   k->args.addr = gpr_malloc(args->addr_len);
-  memcpy(k->args.addr, args->addr, k->args.addr_len);
+  if (k->args.addr_len > 0) {
+    memcpy(k->args.addr, args->addr, k->args.addr_len);
+  }
   k->args.args = copy_channel_args(args->args);
   return k;
 }
@@ -104,11 +111,15 @@ static int subchannel_key_compare(grpc_subchannel_key *a,
   if (c != 0) return c;
   c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
   if (c != 0) return c;
-  c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
+  if (a->args.addr_len) {
-  if (c != 0) return c;
+    c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
-  c = memcmp(a->args.filters, b->args.filters,
+    if (c != 0) return c;
-             a->args.filter_count * sizeof(*a->args.filters));
+  }
-  if (c != 0) return c;
+  if (a->args.filter_count > 0) {
+    c = memcmp(a->args.filters, b->args.filters,
+               a->args.filter_count * sizeof(*a->args.filters));
+    if (c != 0) return c;
+  }
   return grpc_channel_args_compare(a->args.args, b->args.args);
 }
 

src/core/ext/transport/chttp2/transport/frame_goaway.c

@@ -142,7 +142,8 @@ grpc_error *grpc_chttp2_goaway_parser_parse(
       ++cur;
     /* fallthrough */
     case GRPC_CHTTP2_GOAWAY_DEBUG:
-      memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
+      if (end != cur)
+        memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
       GPR_ASSERT((size_t)(end - cur) < UINT32_MAX - p->debug_pos);
       p->debug_pos += (uint32_t)(end - cur);
       p->state = GRPC_CHTTP2_GOAWAY_DEBUG;

src/core/ext/transport/chttp2/transport/hpack_parser.c

@@ -1194,6 +1194,7 @@ static grpc_error *parse_string_prefix(grpc_chttp2_hpack_parser *p,
 /* append some bytes to a string */
 static void append_bytes(grpc_chttp2_hpack_parser_string *str,
                          const uint8_t *data, size_t length) {
+  if (length == 0) return;
   if (length + str->length > str->capacity) {
     GPR_ASSERT(str->length + length <= UINT32_MAX);
     str->capacity = (uint32_t)(str->length + length);
@@ -1518,6 +1519,11 @@ grpc_error *grpc_chttp2_header_parser_parse(
        stream id on a header */
     if (stream_parsing != NULL) {
       if (parser->is_boundary) {
+        if (stream_parsing->header_frames_received ==
+            GPR_ARRAY_SIZE(stream_parsing->got_metadata_on_parse)) {
+          gpr_log(GPR_ERROR, "too many trailer frames");
+          return GRPC_CHTTP2_CONNECTION_ERROR;
+        }
         stream_parsing
             ->got_metadata_on_parse[stream_parsing->header_frames_received] = 1;
         stream_parsing->header_frames_received++;

src/core/lib/channel/channel_args.c

@@ -132,7 +132,8 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
   for (size_t i = 0; i < a->num_args; i++) {
     args[i] = &a->args[i];
   }
-  qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
+  if (a->num_args > 1)
+    qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
 
   grpc_channel_args *b = gpr_malloc(sizeof(grpc_channel_args));
   b->num_args = a->num_args;

src/core/lib/compression/compression_algorithm.c

@@ -199,5 +199,6 @@ void grpc_compression_options_disable_algorithm(
 int grpc_compression_options_is_algorithm_enabled(
     const grpc_compression_options *opts,
     grpc_compression_algorithm algorithm) {
+  if (algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) return 0;
   return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
 }

src/core/lib/support/murmur_hash.c

@@ -33,6 +33,8 @@
 
 #include "src/core/lib/support/murmur_hash.h"
 
+#include <string.h>
+
 #define ROTL32(x, r) ((x) << (r)) | ((x) >> (32 - (r)))
 
 #define FMIX32(h)    \
@@ -42,10 +44,6 @@
   (h) *= 0xc2b2ae35; \
   (h) ^= (h) >> 16;
 
-/* Block read - if your platform needs to do endian-swapping or can only
-   handle aligned reads, do the conversion here */
-#define GETBLOCK32(p, i) (p)[(i)]
-
 uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
   const uint8_t *data = (const uint8_t *)key;
   const size_t nblocks = len / 4;
@@ -62,7 +60,7 @@ uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
 
   /* body */
   for (i = -(int)nblocks; i; i++) {
-    k1 = GETBLOCK32(blocks, i);
+    memcpy(&k1, blocks + i, sizeof(uint32_t));
 
     k1 *= c1;
     k1 = ROTL32(k1, 15);

src/core/lib/transport/metadata.c

@@ -373,7 +373,8 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) {
     ss = g_static_strtab[idx];
     if (ss == NULL) break;
     if (ss->hash == hash && GPR_SLICE_LENGTH(ss->slice) == length &&
-        0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length)) {
+        (length == 0 ||
+         0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length))) {
       GPR_TIMER_END("grpc_mdstr_from_buffer", 0);
       return ss;
     }

src/python/grpcio/grpc/framework/foundation/future.py

@@ -232,6 +232,6 @@ class Future(six.with_metaclass(abc.ABCMeta)):
     immediately.
 
     Args:
-      fn: A callable taking a this Future object as its single parameter.
+      fn: A callable taking this Future object as its single parameter.
     """
     raise NotImplementedError()

test/core/end2end/fuzzers/api_fuzzer.c

@@ -429,15 +429,19 @@ static void add_to_free(call_state *call, void *p) {
 static void read_metadata(input_stream *inp, size_t *count,
                           grpc_metadata **metadata, call_state *cs) {
   *count = next_byte(inp);
-  *metadata = gpr_malloc(*count * sizeof(**metadata));
+  if (*count) {
-  memset(*metadata, 0, *count * sizeof(**metadata));
+    *metadata = gpr_malloc(*count * sizeof(**metadata));
-  for (size_t i = 0; i < *count; i++) {
+    memset(*metadata, 0, *count * sizeof(**metadata));
-    (*metadata)[i].key = read_string(inp);
+    for (size_t i = 0; i < *count; i++) {
-    read_buffer(inp, (char **)&(*metadata)[i].value,
+      (*metadata)[i].key = read_string(inp);
-                &(*metadata)[i].value_length);
+      read_buffer(inp, (char **)&(*metadata)[i].value,
-    (*metadata)[i].flags = read_uint32(inp);
+                  &(*metadata)[i].value_length);
-    add_to_free(cs, (void *)(*metadata)[i].key);
+      (*metadata)[i].flags = read_uint32(inp);
-    add_to_free(cs, (void *)(*metadata)[i].value);
+      add_to_free(cs, (void *)(*metadata)[i].key);
+      add_to_free(cs, (void *)(*metadata)[i].value);
+    }
+  } else {
+    *metadata = gpr_malloc(1);
   }
   add_to_free(cs, *metadata);
 }

tools/run_tests/configs.json

@@ -57,7 +57,7 @@
   {
     "config": "ubsan", 
     "environ": {
-      "UBSAN_OPTIONS": "print_stacktrace=1"
+      "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1"
     }, 
     "timeout_multiplier": 1.5
   }, 

tools/run_tests/performance/scenario_config.py

@@ -94,6 +94,7 @@ def remove_nonproto_fields(scenario):
 def _ping_pong_scenario(name, rpc_type,
                         client_type, server_type,
                         secure=True,
+                        use_big_generic_payload=False,
                         use_generic_payload=False,
                         unconstrained_client=None,
                         client_language=None,
@@ -128,7 +129,12 @@ def _ping_pong_scenario(name, rpc_type,
     'warmup_seconds': warmup_seconds,
     'benchmark_seconds': BENCHMARK_SECONDS
   }
-  if use_generic_payload:
+  if use_big_generic_payload:
+    if server_type != 'ASYNC_GENERIC_SERVER':
+      raise Exception('Use ASYNC_GENERIC_SERVER for big generic payload.')
+    scenario['client_config']['payload_config'] = BIG_GENERIC_PAYLOAD
+    scenario['server_config']['payload_config'] = BIG_GENERIC_PAYLOAD    
+  elif use_generic_payload:
     if server_type != 'ASYNC_GENERIC_SERVER':
       raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
     scenario['client_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD
@@ -141,13 +147,23 @@ def _ping_pong_scenario(name, rpc_type,
     if unconstrained_client == 'async':
       deep = DEEP
       wide = WIDE
+      num_clients = 0 # use as many clients as available
     elif unconstrained_client == 'sync':
       deep = SYNC_DEEP
       wide = SYNC_WIDE
+      num_clients = 0 # use as many clients as available
+    elif unconstrained_client == '1chan_bw':
+      deep = DEEP
+      wide = 1
+      num_clients = 1 # limit to 1 for a single channel
+    elif unconstrained_client == 'Nchan_bw':
+      deep = DEEP
+      wide = WIDE
+      num_clients = 0 # use as many clients as available
     else:
       raise Exception('Illegal value of unconstrained_client option.')
 
-    scenario['num_clients'] = 0  # use as many client as available.
+    scenario['num_clients'] = num_clients
     scenario['client_config']['outstanding_rpcs_per_channel'] = deep
     scenario['client_config']['client_channels'] = wide
     scenario['client_config']['async_client_threads'] = 0
@@ -238,6 +254,19 @@ class CXXLanguage:
           server_core_limit=1, async_server_threads=1,
           secure=secure)
 
+      yield _ping_pong_scenario(
+          'cpp_generic_async_streaming_single_channel_throughput_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          unconstrained_client='1chan_bw', use_big_generic_payload=True,
+          secure=secure,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'cpp_generic_async_streaming_multi_channel_throughput_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          unconstrained_client='Nchan_bw', use_big_generic_payload=True,
+          secure=secure)
+
   def __str__(self):
     return 'c++'
 

tools/run_tests/tests.json

@@ -23227,6 +23227,58 @@
     ], 
     "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure"
   }, 
+  {
+    "args": [
+      "--scenario_json", 
+      "'{\"name\": \"cpp_generic_async_streaming_single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+    ], 
+    "boringssl": true, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1000.0, 
+    "defaults": "boringssl", 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
+    "name": "json_run_localhost", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_single_channel_throughput_secure"
+  }, 
+  {
+    "args": [
+      "--scenario_json", 
+      "'{\"name\": \"cpp_generic_async_streaming_multi_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+    ], 
+    "boringssl": true, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1000.0, 
+    "defaults": "boringssl", 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
+    "name": "json_run_localhost", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_multi_channel_throughput_secure"
+  }, 
   {
     "args": [
       "--scenario_json", 
@@ -23435,6 +23487,58 @@
     ], 
     "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure"
   }, 
+  {
+    "args": [
+      "--scenario_json", 
+      "'{\"name\": \"cpp_generic_async_streaming_single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+    ], 
+    "boringssl": true, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1000.0, 
+    "defaults": "boringssl", 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
+    "name": "json_run_localhost", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_single_channel_throughput_insecure"
+  }, 
+  {
+    "args": [
+      "--scenario_json", 
+      "'{\"name\": \"cpp_generic_async_streaming_multi_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+    ], 
+    "boringssl": true, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1000.0, 
+    "defaults": "boringssl", 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
+    "name": "json_run_localhost", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_multi_channel_throughput_insecure"
+  }, 
   {
     "args": [
       "test/core/end2end/fuzzers/api_fuzzer_corpus/00.bin"