Merge branch 'master' of github.com:grpc/grpc

include/grpc++/dynamic_thread_pool.h

@@ -41,6 +41,7 @@
 #include <grpc++/thread_pool_interface.h>
 
 #include <list>
+#include <memory>
 #include <queue>
 
 namespace grpc {

include/grpc++/impl/sync_no_cxx11.h

@@ -87,7 +87,7 @@ class condition_variable {
   ~condition_variable() { gpr_cv_destroy(&cv_); }
   void wait(lock_guard<mutex> &mu) {
     mu.locked = false;
-    gpr_cv_wait(&cv_, &mu.mu_.mu_, gpr_inf_future(GPR_CLOCK_REALTIME);
+    gpr_cv_wait(&cv_, &mu.mu_.mu_, gpr_inf_future(GPR_CLOCK_REALTIME));
     mu.locked = true;
   }
   void notify_one() { gpr_cv_signal(&cv_); }

include/grpc/census.h

@@ -44,26 +44,30 @@
 extern "C" {
 #endif
 
-/* Identify census functionality that can be enabled via census_initialize(). */
-enum census_functions {
-  CENSUS_NONE = 0,    /* Do not enable census. */
-  CENSUS_TRACING = 1, /* Enable census tracing. */
-  CENSUS_STATS = 2,   /* Enable Census stats collection. */
-  CENSUS_CPU = 4,     /* Enable Census CPU usage collection. */
-  CENSUS_ALL = CENSUS_TRACING | CENSUS_STATS | CENSUS_CPU
+/* Identify census features that can be enabled via census_initialize(). */
+enum census_features {
+  CENSUS_FEATURE_NONE = 0,    /* Do not enable census. */
+  CENSUS_FEATURE_TRACING = 1, /* Enable census tracing. */
+  CENSUS_FEATURE_STATS = 2,   /* Enable Census stats collection. */
+  CENSUS_FEATURE_CPU = 4,     /* Enable Census CPU usage collection. */
+  CENSUS_FEATURE_ALL =
+      CENSUS_FEATURE_TRACING | CENSUS_FEATURE_STATS | CENSUS_FEATURE_CPU
 };
 
-/* Shutdown and startup census subsystem. The 'functions' argument should be
- * the OR (|) of census_functions values. If census fails to initialize, then
+/** Shutdown and startup census subsystem. The 'features' argument should be
+ * the OR (|) of census_features values. If census fails to initialize, then
  * census_initialize() will return a non-zero value. It is an error to call
  * census_initialize() more than once (without an intervening
  * census_shutdown()). */
-int census_initialize(int functions);
-void census_shutdown();
+int census_initialize(int features);
+void census_shutdown(void);
 
-/* If any census feature has been initialized, this funtion will return a
- * non-zero value. */
-int census_available();
+/** Return the features supported by the current census implementation (not all
+ * features will be available on all platforms). */
+int census_supported(void);
+
+/** Return the census features currently enabled. */
+int census_enabled(void);
 
 /* Internally, Census relies on a context, which should be propagated across
  * RPC's. From the RPC subsystems viewpoint, this is an opaque data structure.

src/core/census/grpc_context.c

@@ -39,7 +39,7 @@ static void grpc_census_context_destroy(void *context) {
 }
 
 void grpc_census_call_set_context(grpc_call *call, census_context *context) {
-  if (!census_available()) {
+  if (census_enabled() == CENSUS_FEATURE_NONE) {
     return;
   }
   if (context == NULL) {

src/core/census/initialize.c

@@ -33,20 +33,25 @@
 
 #include <grpc/census.h>
 
-static int census_fns_enabled = CENSUS_NONE;
+static int features_enabled = CENSUS_FEATURE_NONE;
 
-int census_initialize(int functions) {
-  if (census_fns_enabled != CENSUS_NONE) {
+int census_initialize(int features) {
+  if (features_enabled != CENSUS_FEATURE_NONE) {
     return 1;
   }
-  if (functions != CENSUS_NONE) {
+  if (features != CENSUS_FEATURE_NONE) {
     return 1;
   } else {
-    census_fns_enabled = functions;
+    features_enabled = features;
     return 0;
   }
 }
 
-void census_shutdown() { census_fns_enabled = CENSUS_NONE; }
+void census_shutdown(void) { features_enabled = CENSUS_FEATURE_NONE; }
 
-int census_available() { return (census_fns_enabled != CENSUS_NONE); }
+int census_supported(void) {
+  /* TODO(aveitch): improve this as we implement features... */
+  return CENSUS_FEATURE_NONE;
+}
+
+int census_enabled(void) { return features_enabled; }

src/core/iomgr/tcp_server_windows.c

@@ -79,6 +79,8 @@ struct grpc_tcp_server {
 
   /* active port count: how many ports are actually still listening */
   int active_ports;
+  /* number of iomgr callbacks that have been explicitly scheduled during shutdown */
+  int iomgr_callbacks_pending;
 
   /* all listening ports */
   server_port *ports;
@@ -93,6 +95,7 @@ grpc_tcp_server *grpc_tcp_server_create(void) {
   gpr_mu_init(&s->mu);
   gpr_cv_init(&s->cv);
   s->active_ports = 0;
+  s->iomgr_callbacks_pending = 0;
   s->cb = NULL;
   s->cb_arg = NULL;
   s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP);
@@ -112,10 +115,10 @@ void grpc_tcp_server_destroy(grpc_tcp_server *s,
   for (i = 0; i < s->nports; i++) {
     server_port *sp = &s->ports[i];
     sp->shutting_down = 1;
-    grpc_winsocket_shutdown(sp->socket);
+    s->iomgr_callbacks_pending += grpc_winsocket_shutdown(sp->socket);
   }
   /* This happens asynchronously. Wait while that happens. */
-  while (s->active_ports) {
+  while (s->active_ports || s->iomgr_callbacks_pending) {
     gpr_cv_wait(&s->cv, &s->mu, gpr_inf_future(GPR_CLOCK_REALTIME));
   }
   gpr_mu_unlock(&s->mu);
@@ -254,8 +257,16 @@ static void on_accept(void *arg, int from_iocp) {
 
   /* The general mechanism for shutting down is to queue abortion calls. While
      this is necessary in the read/write case, it's useless for the accept
-     case. Let's do nothing. */
-  if (!from_iocp) return;
+     case. We only need to adjust the pending callback count */
+  if (!from_iocp) {
+    gpr_mu_lock(&sp->server->mu);
+    GPR_ASSERT(sp->server->iomgr_callbacks_pending > 0);
+    if (0 == --sp->server->iomgr_callbacks_pending) {
+      gpr_cv_broadcast(&sp->server->cv);
+    }
+    gpr_mu_unlock(&sp->server->mu);
+    return;
+  }
 
   /* The IOCP notified us of a completed operation. Let's grab the results,
       and act accordingly. */
@@ -264,11 +275,12 @@ static void on_accept(void *arg, int from_iocp) {
                                             &transfered_bytes, FALSE, &flags);
   if (!wsa_success) {
     if (sp->shutting_down) {
-      /* During the shutdown case, we ARE expecting an error. So that's swell,
+      /* During the shutdown case, we ARE expecting an error. So that's well,
          and we can wake up the shutdown thread. */
       sp->shutting_down = 0;
       sp->socket->read_info.outstanding = 0;
       gpr_mu_lock(&sp->server->mu);
+      GPR_ASSERT(sp->server->active_ports > 0);
       if (0 == --sp->server->active_ports) {
         gpr_cv_broadcast(&sp->server->cv);
       }

src/core/iomgr/tcp_windows.c

@@ -368,7 +368,14 @@ static grpc_endpoint_write_status win_write(grpc_endpoint *ep,
   return GRPC_ENDPOINT_WRITE_PENDING;
 }
 
-static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) {
+static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *ps) {
+  (void) ps;
+  grpc_tcp *tcp = (grpc_tcp *) ep;
+  grpc_iocp_add_socket(tcp->socket);
+}
+
+static void win_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pss) {
+  (void) pss;
   grpc_tcp *tcp = (grpc_tcp *) ep;
   grpc_iocp_add_socket(tcp->socket);
 }
@@ -402,8 +409,9 @@ static char *win_get_peer(grpc_endpoint *ep) {
 }
 
 static grpc_endpoint_vtable vtable = {win_notify_on_read, win_write,
-                                      win_add_to_pollset, win_shutdown,
-                                      win_destroy,        win_get_peer};
+                                      win_add_to_pollset, win_add_to_pollset_set,
+                                      win_shutdown,       win_destroy,
+                                      win_get_peer};
 
 grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *) gpr_malloc(sizeof(grpc_tcp));

src/core/surface/init.c

@@ -80,8 +80,11 @@ void grpc_init(void) {
     grpc_security_pre_init();
     grpc_iomgr_init();
     grpc_tracer_init("GRPC_TRACE");
-    if (census_initialize(CENSUS_NONE)) {
-      gpr_log(GPR_ERROR, "Could not initialize census.");
+    /* Only initialize census if noone else has. */
+    if (census_enabled() == CENSUS_FEATURE_NONE) {
+      if (census_initialize(census_supported())) { /* enable all features. */
+        gpr_log(GPR_ERROR, "Could not initialize census.");
+      }
     }
     grpc_timers_global_init();
   }

tools/gce_setup/cloud_prod_runner.sh

@@ -34,8 +34,8 @@ log_link=https://pantheon.corp.google.com/m/cloudstorage/b/stoked-keyword-656-ou
 
 main() {
   source grpc_docker.sh
-  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response)
-  auth_test_cases=(service_account_creds compute_engine_creds jwt_token_creds)
+  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server)
+  auth_test_cases=(service_account_creds compute_engine_creds jwt_token_creds oauth2_auth_token per_rpc_creds)
   clients=(cxx java go ruby node csharp_mono csharp_dotnet python php)
   for test_case in "${test_cases[@]}"
   do

tools/gce_setup/grpc_docker.sh

@@ -508,7 +508,12 @@ grpc_cloud_prod_auth_test_args() {
   grpc_gen_test_cmd="grpc_cloud_prod_auth_"
   [[ -n $1 ]] && {  # test_case
     test_case=$1
-    grpc_gen_test_cmd+="$1"
+    test_command="service_account_creds"
+    if [ "$test_case" == "compute_engine_creds" ]
+    then
+      test_command="compute_engine_creds"
+    fi
+    grpc_gen_test_cmd+=$test_command
     shift
   } || {
     echo "$FUNCNAME: missing arg: test_case" 1>&2

tools/gce_setup/interop_test_runner.sh

@@ -37,7 +37,7 @@ fail_log_link=https://pantheon.corp.google.com/m/cloudstorage/b/stoked-keyword-6
 
 main() {
   source grpc_docker.sh
-  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response)
+  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server)
   clients=(cxx java go ruby node python csharp_mono php)
   servers=(cxx java go ruby node python csharp_mono)
   for test_case in "${test_cases[@]}"

tools/jenkins/run_jenkins.sh

@@ -92,7 +92,9 @@ then
   docker cp $DOCKER_CID:/var/local/git/grpc/report.xml $git_root
   sleep 4
   docker rm $DOCKER_CID || true
-
+elif [ "$platform" == "interop" ]
+then
+  python tools/run_tests/run_interops.py --language=$language
 elif [ "$platform" == "windows" ]
 then
   echo "building $language on Windows"

tools/run_tests/jobset.py

@@ -206,7 +206,7 @@ class Job(object):
                 do_newline=self._newline_on_success or self._travis)
         if self._bin_hash:
           update_cache.finished(self._spec.identity(), self._bin_hash)
-    elif self._state == _RUNNING and time.time() - self._start > 600:
+    elif self._state == _RUNNING and time.time() - self._start > 900:
       self._tempfile.seek(0)
       stdout = self._tempfile.read()
       filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))

tools/run_tests/run_interops.py

@@ -0,0 +1,36 @@
+import argparse
+import xml.etree.cElementTree as ET
+import jobset
+
+argp = argparse.ArgumentParser(description='Run interop tests.')
+argp.add_argument('-l', '--language',
+                  choices=['build_only', 'c++'],
+                  nargs='+',
+                  default=['build_only'])
+args = argp.parse_args()
+
+# build job
+build_steps = 'tools/run_tests/run_interops_build.sh'
+build_job = jobset.JobSpec(cmdline=build_steps, shortname='build')
+
+# test jobs
+_TESTS = ['large_unary', 'empty_unary', 'ping_pong', 'client_streaming', 'server_streaming']
+jobs = []
+jobNumber = 0
+for lang in args.language:
+  for test in _TESTS:
+    test_job = jobset.JobSpec(cmdline=['tools/run_tests/run_interops_test.sh', '%s' % lang, '%s' % test], shortname=test)
+    jobs.append(test_job)
+    jobNumber+=1
+
+root = ET.Element('testsuites')
+testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests')
+
+# always do the build of docker first, and then all the tests can run in parallel
+jobset.run([build_job], maxjobs=1, xml_report=testsuite)
+jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite)
+
+tree = ET.ElementTree(root)
+tree.write('report.xml', encoding='UTF-8')
+
+

tools/run_tests/run_interops_build.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -e
+
+#clean up any old docker files
+sudo docker rmi -f grpc/cxx || true
+sudo docker rmi -f grpc/base || true
+sudo docker rmi -f 0.0.0.0:5000/grpc/base || true
+
+#prepare building by pulling down base images and necessary files
+sudo docker pull 0.0.0.0:5000/grpc/base
+sudo docker tag -f 0.0.0.0:5000/grpc/base grpc/base
+gsutil cp -R gs://docker-interop-images/admin/service_account tools/dockerfile/grpc_cxx
+gsutil cp -R gs://docker-interop-images/admin/cacerts tools/dockerfile/grpc_cxx
+
+#build docker file, add more languages later
+sudo docker build --no-cache -t grpc/cxx tools/dockerfile/grpc_cxx

tools/run_tests/run_interops_test.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+language=$1
+test_case=$2
+
+set -e
+if [ "$language" = "c++" ]
+then
+  sudo docker run grpc/cxx /var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots --server_host_override=grpc-test.sandbox.google.com --server_host=grpc-test.sandbox.google.com --server_port=443 --test_case=$test_case
+else
+  echo "interop testss not added for $language"
+  exit 1
+fi
+