Merge remote-tracking branch 'upstream/master'

Makefile

@@ -257,7 +257,7 @@ HAS_SYSTEM_ZLIB = false
 HAS_SYSTEM_PROTOBUF = false
 endif
 
-HAS_PROTOC = $(shell $(PROTOC_CMD) && echo true || echo false)
+HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false)
 ifeq ($(HAS_PROTOC),true)
 HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false)
 else
@@ -2081,7 +2081,7 @@ install-certs: etc/roots.pem
 	$(Q) $(INSTALL) etc/roots.pem $(prefix)/share/grpc/roots.pem
 
 verify-install:
-ifeq ($(SYSTEM_OK),true)
+ifeq ($(INSTALL_OK),true)
 	@echo "Your system looks ready to go."
 	@echo
 else

include/grpc++/server.h

@@ -69,6 +69,11 @@ class Server final : private CallHook,
   // Shutdown the server, block until all rpc processing finishes.
   void Shutdown();
 
+  // Block waiting for all work to complete (the server must either
+  // be shutting down or some other thread must call Shutdown for this
+  // function to ever return)
+  void Wait();
+
  private:
   friend class ServerBuilder;
 

include/grpc/support/alloc.h

@@ -46,8 +46,8 @@ void *gpr_malloc(size_t size);
 void gpr_free(void *ptr);
 /* realloc, never returns NULL */
 void *gpr_realloc(void *p, size_t size);
-/* aligned malloc, never returns NULL, alignment must be power of 2 */
+/* aligned malloc, never returns NULL, will align to 1 << alignment_log */
-void *gpr_malloc_aligned(size_t size, size_t alignment);
+void *gpr_malloc_aligned(size_t size, size_t alignment_log);
 /* free memory allocated by gpr_malloc_aligned */
 void gpr_free_aligned(void *ptr);
 

include/grpc/support/atm_win32.h

@@ -93,11 +93,13 @@ static __inline gpr_atm gpr_atm_no_barrier_fetch_add(gpr_atm *p,
 static __inline gpr_atm gpr_atm_full_fetch_add(gpr_atm *p, gpr_atm delta) {
   /* Use a CAS operation to get pointer-sized fetch and add */
   gpr_atm old;
+#ifdef GPR_ARCH_64
   do {
     old = *p;
-#ifdef GPR_ARCH_64
   } while (old != (gpr_atm)InterlockedCompareExchange64(p, old + delta, old));
 #else
+  do {
+    old = *p;
   } while (old != (gpr_atm)InterlockedCompareExchange(p, old + delta, old));
 #endif
   return old;

include/grpc/support/port_platform.h

@@ -147,16 +147,18 @@
 #include <stdint.h>
 
 /* Cache line alignment */
-#ifndef GPR_CACHELINE_SIZE
+#ifndef GPR_CACHELINE_SIZE_LOG
 #if defined(__i386__) || defined(__x86_64__)
-#define GPR_CACHELINE_SIZE 64
+#define GPR_CACHELINE_SIZE_LOG 6
 #endif
-#ifndef GPR_CACHELINE_SIZE
+#ifndef GPR_CACHELINE_SIZE_LOG
 /* A reasonable default guess. Note that overestimates tend to waste more
    space, while underestimates tend to waste more time. */
-#define GPR_CACHELINE_SIZE 64
+#define GPR_CACHELINE_SIZE_LOG 6
-#endif /* GPR_CACHELINE_SIZE */
+#endif /* GPR_CACHELINE_SIZE_LOG */
-#endif /* GPR_CACHELINE_SIZE */
+#endif /* GPR_CACHELINE_SIZE_LOG */
+
+#define GPR_CACHELINE_SIZE (1 << GPR_CACHELINE_SIZE_LOG)
 
 /* scrub GCC_ATOMIC if it's not available on this compiler */
 #if defined(GPR_GCC_ATOMIC) && !defined(__ATOMIC_RELAXED)

include/grpc/support/slice.h

@@ -165,7 +165,9 @@ gpr_slice gpr_slice_split_head(gpr_slice *s, size_t split);
 
 gpr_slice gpr_empty_slice(void);
 
-/* Returns <0 if a < b, ==0 if a == b, >0 if a > b */
+/* Returns <0 if a < b, ==0 if a == b, >0 if a > b
+   The order is arbitrary, and is not guaranteed to be stable across different
+   versions of the API. */
 int gpr_slice_cmp(gpr_slice a, gpr_slice b);
 int gpr_slice_str_cmp(gpr_slice a, const char *b);
 

src/core/compression/message_compress.c

@@ -48,7 +48,6 @@ static int zlib_body(z_stream *zs, gpr_slice_buffer *input,
   int r;
   int flush;
   size_t i;
-  size_t output_bytes = 0;
   gpr_slice outbuf = gpr_slice_malloc(OUTPUT_BLOCK_SIZE);
 
   zs->avail_out = GPR_SLICE_LENGTH(outbuf);
@@ -60,7 +59,6 @@ static int zlib_body(z_stream *zs, gpr_slice_buffer *input,
     zs->next_in = GPR_SLICE_START_PTR(input->slices[i]);
     do {
       if (zs->avail_out == 0) {
-        output_bytes += GPR_SLICE_LENGTH(outbuf);
         gpr_slice_buffer_add_indexed(output, outbuf);
         outbuf = gpr_slice_malloc(OUTPUT_BLOCK_SIZE);
         zs->avail_out = GPR_SLICE_LENGTH(outbuf);
@@ -80,7 +78,6 @@ static int zlib_body(z_stream *zs, gpr_slice_buffer *input,
 
   GPR_ASSERT(outbuf.refcount);
   outbuf.data.refcounted.length -= zs->avail_out;
-  output_bytes += GPR_SLICE_LENGTH(outbuf);
   gpr_slice_buffer_add_indexed(output, outbuf);
 
   return 1;

src/core/statistics/census_log.c

@@ -475,11 +475,11 @@ void census_log_initialize(size_t size_in_mb, int discard_old_records) {
   g_log.block_being_read = NULL;
   gpr_atm_rel_store(&g_log.is_full, 0);
   g_log.core_local_blocks = (cl_core_local_block*)gpr_malloc_aligned(
-      g_log.num_cores * sizeof(cl_core_local_block), GPR_CACHELINE_SIZE);
+      g_log.num_cores * sizeof(cl_core_local_block), GPR_CACHELINE_SIZE_LOG);
   memset(g_log.core_local_blocks, 0,
          g_log.num_cores * sizeof(cl_core_local_block));
   g_log.blocks = (cl_block*)gpr_malloc_aligned(
-      g_log.num_blocks * sizeof(cl_block), GPR_CACHELINE_SIZE);
+      g_log.num_blocks * sizeof(cl_block), GPR_CACHELINE_SIZE_LOG);
   memset(g_log.blocks, 0, g_log.num_blocks * sizeof(cl_block));
   g_log.buffer = gpr_malloc(g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);
   memset(g_log.buffer, 0, g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);

src/core/support/alloc.c

@@ -54,7 +54,8 @@ void *gpr_realloc(void *p, size_t size) {
   return p;
 }
 
-void *gpr_malloc_aligned(size_t size, size_t alignment) {
+void *gpr_malloc_aligned(size_t size, size_t alignment_log) {
+  size_t alignment = 1 << alignment_log;
   size_t extra = alignment - 1 + sizeof(void *);
   void *p = gpr_malloc(size + extra);
   void **ret = (void **)(((gpr_uintptr)p + extra) & ~(alignment - 1));

src/core/support/cpu_linux.c

@@ -39,25 +39,28 @@
 
 #ifdef GPR_CPU_LINUX
 
-#include <grpc/support/cpu.h>
-
 #include <sched.h>
 #include <errno.h>
 #include <unistd.h>
 #include <string.h>
 
+#include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
+#include <grpc/support/sync.h>
 
-unsigned gpr_cpu_num_cores(void) {
+static int ncpus = 0;
-  static int ncpus = 0;
+
-  /* FIXME: !threadsafe */
+static void init_num_cpus() {
-  if (ncpus == 0) {
+  ncpus = sysconf(_SC_NPROCESSORS_ONLN);
-    ncpus = sysconf(_SC_NPROCESSORS_ONLN);
+  if (ncpus < 1) {
-    if (ncpus < 1) {
+    gpr_log(GPR_ERROR, "Cannot determine number of CPUs: assuming 1");
-      gpr_log(GPR_ERROR, "Cannot determine number of CPUs: assuming 1");
+    ncpus = 1;
-      ncpus = 1;
-    }
   }
+}
+
+unsigned gpr_cpu_num_cores(void) {
+  static gpr_once once = GPR_ONCE_INIT;
+  gpr_once_init(&once, init_num_cpus);
   return ncpus;
 }
 

src/core/support/cpu_posix.c

@@ -43,15 +43,19 @@
 
 static __thread char magic_thread_local;
 
-unsigned gpr_cpu_num_cores(void) {
+static int ncpus = 0;
-  static int ncpus = 0;
+
-  if (ncpus == 0) {
+static void init_ncpus() {
-    ncpus = sysconf(_SC_NPROCESSORS_ONLN);
+  ncpus = sysconf(_SC_NPROCESSORS_ONLN);
-    if (ncpus < 1) {
+  if (ncpus < 1) {
-      gpr_log(GPR_ERROR, "Cannot determine number of CPUs: assuming 1");
+    gpr_log(GPR_ERROR, "Cannot determine number of CPUs: assuming 1");
-      ncpus = 1;
+    ncpus = 1;
-    }
   }
+}
+
+unsigned gpr_cpu_num_cores(void) {
+  static gpr_once once = GPR_ONCE_INIT;
+  gpr_once_init(&once, init_num_cpus);
   return ncpus;
 }
 

src/core/support/string.c

@@ -91,7 +91,6 @@ char *gpr_hexdump(const char *buf, size_t len, gpr_uint32 flags) {
   }
 
   if (flags & GPR_HEXDUMP_PLAINTEXT) {
-    cur = beg;
     if (len) hexout_append(&out, ' ');
     hexout_append(&out, '\'');
     for (cur = beg; cur != end; ++cur) {

src/core/tsi/ssl_transport_security.c

@@ -1150,6 +1150,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory(
       if (result != TSI_OK) {
         gpr_log(GPR_ERROR, "Building alpn list failed with error %s.",
                 tsi_result_to_string(result));
+        free(alpn_protocol_list);
         break;
       }
       ssl_failed = SSL_CTX_set_alpn_protos(ssl_context, alpn_protocol_list,

src/cpp/server/server.cc

@@ -265,21 +265,26 @@ bool Server::Start() {
 }
 
 void Server::Shutdown() {
-  {
+  std::unique_lock<std::mutex> lock(mu_);
-    std::unique_lock<std::mutex> lock(mu_);
+  if (started_ && !shutdown_) {
-    if (started_ && !shutdown_) {
+    shutdown_ = true;
-      shutdown_ = true;
+    grpc_server_shutdown(server_);
-      grpc_server_shutdown(server_);
+    cq_.Shutdown();
-      cq_.Shutdown();
 
-      // Wait for running callbacks to finish.
+    // Wait for running callbacks to finish.
-      while (num_running_cb_ != 0) {
+    while (num_running_cb_ != 0) {
-        callback_cv_.wait(lock);
+      callback_cv_.wait(lock);
-      }
     }
   }
 }
 
+void Server::Wait() {
+  std::unique_lock<std::mutex> lock(mu_);
+  while (num_running_cb_ != 0) {
+    callback_cv_.wait(lock);
+  }
+}
+
 void Server::PerformOpsOnCall(CallOpBuffer* buf, Call* call) {
   static const size_t MAX_OPS = 8;
   size_t nops = MAX_OPS;

src/node/ext/credentials.cc

@@ -130,7 +130,7 @@ NAN_METHOD(Credentials::New) {
 
 NAN_METHOD(Credentials::CreateDefault) {
   NanScope();
-  NanReturnValue(WrapStruct(grpc_default_credentials_create()));
+  NanReturnValue(WrapStruct(grpc_google_default_credentials_create()));
 }
 
 NAN_METHOD(Credentials::CreateSsl) {

src/node/interop/messages.proto

@@ -49,7 +49,7 @@ enum PayloadType {
 // A block of data, to simply increase gRPC message size.
 message Payload {
   // The type of data in body.
-  optional PayloadType type = 1;
+  optional PayloadType type = 1 [default = COMPRESSABLE];
   // Primary contents of payload.
   optional bytes body = 2;
 }
@@ -58,7 +58,7 @@ message Payload {
 message SimpleRequest {
   // Desired payload type in the response from the server.
   // If response_type is RANDOM, server randomly chooses one from other formats.
-  optional PayloadType response_type = 1;
+  optional PayloadType response_type = 1 [default = COMPRESSABLE];
 
   // Desired payload size in the response from the server.
   // If response_type is COMPRESSABLE, this denotes the size before compression.
@@ -116,7 +116,7 @@ message StreamingOutputCallRequest {
   // If response_type is RANDOM, the payload from each response in the stream
   // might be of different types. This is to simulate a mixed type of payload
   // stream.
-  optional PayloadType response_type = 1;
+  optional PayloadType response_type = 1 [default = COMPRESSABLE];
 
   // Configuration for each expected response message.
   repeated ResponseParameters response_parameters = 2;

src/node/src/client.js

@@ -245,7 +245,9 @@ function makeUnaryRequestFunction(method, serialize, deserialize) {
           return;
         }
         if (response.status.code !== grpc.status.OK) {
-          callback(response.status);
+          var error = new Error(response.status.details);
+          error.code = response.status.code;
+          callback(error);
           return;
         }
         emitter.emit('status', response.status);
@@ -314,7 +316,9 @@ function makeClientStreamRequestFunction(method, serialize, deserialize) {
           return;
         }
         if (response.status.code !== grpc.status.OK) {
-          callback(response.status);
+          var error = new Error(response.status.details);
+          error.code = response.status.code;
+          callback(error);
           return;
         }
         stream.emit('status', response.status);

src/python/src/grpc/_adapter/rear.py

@@ -170,7 +170,8 @@ class RearLink(ticket_interfaces.RearLink, activated.Activated):
     if event.status.code is _low.Code.OK:
       category = tickets.Kind.COMPLETION
     elif event.status.code is _low.Code.CANCELLED:
-      category = tickets.Kind.CANCELLATION
+      # TODO(issue 752): Use a CANCELLATION ticket kind here.
+      category = tickets.Kind.SERVICER_FAILURE
     elif event.status.code is _low.Code.EXPIRED:
       category = tickets.Kind.EXPIRATION
     else:

src/ruby/ext/grpc/rb_credentials.c

@@ -125,7 +125,7 @@ static VALUE grpc_rb_credentials_init_copy(VALUE copy, VALUE orig) {
     Creates the default credential instances. */
 static VALUE grpc_rb_default_credentials_create(VALUE cls) {
   grpc_rb_credentials *wrapper = ALLOC(grpc_rb_credentials);
-  wrapper->wrapped = grpc_default_credentials_create();
+  wrapper->wrapped = grpc_google_default_credentials_create();
   if (wrapper->wrapped == NULL) {
     rb_raise(rb_eRuntimeError,
              "could not create default credentials, not sure why");

src/ruby/spec/generic/active_call_spec.rb

@@ -67,7 +67,7 @@ describe GRPC::ActiveCall do
     end
 
     describe '#multi_req_view' do
-      it 'exposes a fixed subset of the ActiveCall methods' do
+      xit 'exposes a fixed subset of the ActiveCall methods' do
         want = %w(cancelled, deadline, each_remote_read, shutdown)
         v = @client_call.multi_req_view
         want.each do |w|
@@ -77,7 +77,7 @@ describe GRPC::ActiveCall do
     end
 
     describe '#single_req_view' do
-      it 'exposes a fixed subset of the ActiveCall methods' do
+      xit 'exposes a fixed subset of the ActiveCall methods' do
         want = %w(cancelled, deadline, shutdown)
         v = @client_call.single_req_view
         want.each do |w|

templates/Makefile.template

@@ -274,7 +274,7 @@ HAS_SYSTEM_ZLIB = false
 HAS_SYSTEM_PROTOBUF = false
 endif
 
-HAS_PROTOC = $(shell $(PROTOC_CMD) && echo true || echo false)
+HAS_PROTOC = $(shell $(PROTOC_CMD) > /dev/null && echo true || echo false)
 ifeq ($(HAS_PROTOC),true)
 HAS_VALID_PROTOC = $(shell $(PROTOC_CHECK_CMD) 2> /dev/null && echo true || echo false)
 else
@@ -830,7 +830,7 @@ install-certs: etc/roots.pem
 	$(Q) $(INSTALL) etc/roots.pem $(prefix)/share/grpc/roots.pem
 
 verify-install:
-ifeq ($(SYSTEM_OK),true)
+ifeq ($(INSTALL_OK),true)
 	@echo "Your system looks ready to go."
 	@echo
 else

tools/dockerfile/grpc_build_deb/Dockerfile

@@ -1,3 +1,32 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 # Dockerfile to build Debian packages for gRPC C core.
 FROM grpc/base
 

tools/dockerfile/grpc_ruby_deb/Dockerfile

@@ -1,3 +1,32 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 # Dockerfile for gRPC Ruby, but using Debian packages for gRPC C core.
 FROM grpc/ruby_base
 

tools/run_tests/jobset.py

@@ -33,6 +33,7 @@ import hashlib
 import multiprocessing
 import os
 import random
+import signal
 import subprocess
 import sys
 import tempfile
@@ -42,6 +43,12 @@ import time
 _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
 
 
+# setup a signal handler so that signal.pause registers 'something'
+# when a child finishes
+# not using futures and threading to avoid a dependency on subprocess32
+signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None)
+
+
 def shuffle_iteratable(it):
   """Return an iterable that randomly walks it"""
   # take a random sampling from the passed in iterable
@@ -94,16 +101,19 @@ _TAG_COLOR = {
 
 
 def message(tag, message, explanatory_text=None, do_newline=False):
-  sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % (
+  try:
-      _BEGINNING_OF_LINE,
+    sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % (
-      _CLEAR_LINE,
+        _BEGINNING_OF_LINE,
-      '\n%s' % explanatory_text if explanatory_text is not None else '',
+        _CLEAR_LINE,
-      _COLORS[_TAG_COLOR[tag]][1],
+        '\n%s' % explanatory_text if explanatory_text is not None else '',
-      _COLORS[_TAG_COLOR[tag]][0],
+        _COLORS[_TAG_COLOR[tag]][1],
-      tag,
+        _COLORS[_TAG_COLOR[tag]][0],
-      message,
+        tag,
-      '\n' if do_newline or explanatory_text is not None else ''))
+        message,
-  sys.stdout.flush()
+        '\n' if do_newline or explanatory_text is not None else ''))
+    sys.stdout.flush()
+  except:
+    pass
 
 
 def which(filename):
@@ -232,7 +242,7 @@ class Jobset(object):
       if dead: return
       message('WAITING', '%d jobs running, %d complete, %d failed' % (
           len(self._running), self._completed, self._failures))
-      time.sleep(0.1)
+      signal.pause()
 
   def cancelled(self):
     """Poll for cancellation."""

tools/run_tests/run_tests.py

@@ -36,6 +36,7 @@ import itertools
 import json
 import multiprocessing
 import os
+import re
 import sys
 import time
 
@@ -168,6 +169,7 @@ argp.add_argument('-c', '--config',
                   nargs='+',
                   default=_DEFAULT)
 argp.add_argument('-n', '--runs_per_test', default=1, type=int)
+argp.add_argument('-r', '--regex', default='.*', type=str)
 argp.add_argument('-j', '--jobs', default=1000, type=int)
 argp.add_argument('-f', '--forever',
                   default=False,
@@ -205,7 +207,8 @@ one_run = set(
     spec
     for config in run_configs
     for language in args.language
-    for spec in _LANGUAGES[language].test_specs(config))
+    for spec in _LANGUAGES[language].test_specs(config)
+    if re.search(args.regex, spec.shortname))
 
 runs_per_test = args.runs_per_test
 forever = args.forever