Merge github.com:grpc/grpc into really-really-finish-the-stream

src/core/iomgr/tcp_posix.c

@@ -138,8 +138,10 @@ static void slice_state_remove_prefix(grpc_tcp_slice_state *state,
          native "trim the first N bytes" operation to splice */
       /* TODO(klempner): This really shouldn't be modifying the current slice
          unless we own the slices array. */
-      *current_slice = gpr_slice_split_tail(current_slice, prefix_bytes);
+      gpr_slice tail;
+      tail = gpr_slice_split_tail(current_slice, prefix_bytes);
       gpr_slice_unref(*current_slice);
+      *current_slice = tail;
       return;
     } else {
       gpr_slice_unref(*current_slice);

src/node/src/server.js

@@ -291,7 +291,15 @@ function _read(size) {
       return;
     }
     var data = event.read;
-    if (self.push(self.deserialize(data)) && data !== null) {
+    var deserialized;
+    try {
+      deserialized = self.deserialize(data);
+    } catch (e) {
+      e.code = grpc.status.INVALID_ARGUMENT;
+      self.emit('error', e);
+      return;
+    }
+    if (self.push(deserialized) && data !== null) {
       var read_batch = {};
       read_batch[grpc.opType.RECV_MESSAGE] = true;
       self.call.startBatch(read_batch, readCallback);
@@ -354,7 +362,13 @@ function handleUnary(call, handler, metadata) {
       handleError(call, err);
       return;
     }
-    emitter.request = handler.deserialize(result.read);
+    try {
+      emitter.request = handler.deserialize(result.read);
+    } catch (e) {
+      e.code = grpc.status.INVALID_ARGUMENT;
+      handleError(call, e);
+      return;
+    }
     if (emitter.cancelled) {
       return;
     }
@@ -388,7 +402,13 @@ function handleServerStreaming(call, handler, metadata) {
       stream.emit('error', err);
       return;
     }
-    stream.request = handler.deserialize(result.read);
+    try {
+      stream.request = handler.deserialize(result.read);
+    } catch (e) {
+      e.code = grpc.status.INVALID_ARGUMENT;
+      stream.emit('error', e);
+      return;
+    }
     handler.func(stream);
   });
 }
@@ -401,6 +421,9 @@ function handleServerStreaming(call, handler, metadata) {
  */
 function handleClientStreaming(call, handler, metadata) {
   var stream = new ServerReadableStream(call, handler.deserialize);
+  stream.on('error', function(error) {
+    handleError(call, error);
+  });
   waitForCancel(call, stream);
   var metadata_batch = {};
   metadata_batch[grpc.opType.SEND_INITIAL_METADATA] = metadata;

src/node/test/surface_test.js

@@ -47,6 +47,8 @@ var mathService = math_proto.lookup('math.Math');
 
 var capitalize = require('underscore.string/capitalize');
 
+var _ = require('underscore');
+
 describe('File loader', function() {
   it('Should load a proto file by default', function() {
     assert.doesNotThrow(function() {
@@ -178,9 +180,10 @@ describe('Generic client and server', function() {
     });
   });
 });
-describe('Trailing metadata', function() {
+describe('Other conditions', function() {
   var client;
   var server;
+  var port;
   before(function() {
     var test_proto = ProtoBuf.loadProtoFile(__dirname + '/test_service.proto');
     var test_service = test_proto.lookup('TestService');
@@ -246,7 +249,7 @@ describe('Trailing metadata', function() {
         }
       }
     });
-    var port = server.bind('localhost:0');
+    port = server.bind('localhost:0');
     var Client = surface_client.makeProtobufClientConstructor(test_service);
     client = new Client('localhost:' + port);
     server.listen();
@@ -254,86 +257,167 @@ describe('Trailing metadata', function() {
   after(function() {
     server.shutdown();
   });
-  it('should be present when a unary call succeeds', function(done) {
+  describe('Server recieving bad input', function() {
-    var call = client.unary({error: false}, function(err, data) {
+    var misbehavingClient;
-      assert.ifError(err);
+    var badArg = new Buffer([0xFF]);
+    before(function() {
+      var test_service_attrs = {
+        unary: {
+          path: '/TestService/Unary',
+          requestStream: false,
+          responseStream: false,
+          requestSerialize: _.identity,
+          responseDeserialize: _.identity
+        },
+        clientStream: {
+          path: '/TestService/ClientStream',
+          requestStream: true,
+          responseStream: false,
+          requestSerialize: _.identity,
+          responseDeserialize: _.identity
+        },
+        serverStream: {
+          path: '/TestService/ServerStream',
+          requestStream: false,
+          responseStream: true,
+          requestSerialize: _.identity,
+          responseDeserialize: _.identity
+        },
+        bidiStream: {
+          path: '/TestService/BidiStream',
+          requestStream: true,
+          responseStream: true,
+          requestSerialize: _.identity,
+          responseDeserialize: _.identity
+        }
+      };
+      var Client = surface_client.makeClientConstructor(test_service_attrs,
+                                                        'TestService');
+      misbehavingClient = new Client('localhost:' + port);
     });
-    call.on('status', function(status) {
+    it('should respond correctly to a unary call', function(done) {
-      assert.deepEqual(status.metadata.metadata, ['yes']);
+      misbehavingClient.unary(badArg, function(err, data) {
-      done();
+        assert(err);
+        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        done();
+      });
     });
-  });
+    it('should respond correctly to a client stream', function(done) {
-  it('should be present when a unary call fails', function(done) {
+      var call = misbehavingClient.clientStream(function(err, data) {
-    var call = client.unary({error: true}, function(err, data) {
+        assert(err);
-      assert(err);
+        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        done();
+      });
+      call.write(badArg);
+      // TODO(mlumish): Remove call.end()
+      call.end();
     });
-    call.on('status', function(status) {
+    it('should respond correctly to a server stream', function(done) {
-      assert.deepEqual(status.metadata.metadata, ['yes']);
+      var call = misbehavingClient.serverStream(badArg);
-      done();
+      call.on('data', function(data) {
+        assert.fail(data, null, 'Unexpected data', '===');
+      });
+      call.on('error', function(err) {
+        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        done();
+      });
+    });
+    it('should respond correctly to a bidi stream', function(done) {
+      var call = misbehavingClient.bidiStream();
+      call.on('data', function(data) {
+        assert.fail(data, null, 'Unexpected data', '===');
+      });
+      call.on('error', function(err) {
+        assert.strictEqual(err.code, grpc.status.INVALID_ARGUMENT);
+        done();
+      });
+      call.write(badArg);
+      // TODO(mlumish): Remove call.end()
+      call.end();
     });
   });
-  it('should be present when a client stream call succeeds', function(done) {
+  describe('Trailing metadata', function() {
-    var call = client.clientStream(function(err, data) {
+    it('should be present when a unary call succeeds', function(done) {
-      assert.ifError(err);
+      var call = client.unary({error: false}, function(err, data) {
+        assert.ifError(err);
+      });
+      call.on('status', function(status) {
+        assert.deepEqual(status.metadata.metadata, ['yes']);
+        done();
+      });
     });
-    call.write({error: false});
+    it('should be present when a unary call fails', function(done) {
-    call.write({error: false});
+      var call = client.unary({error: true}, function(err, data) {
-    call.end();
+        assert(err);
-    call.on('status', function(status) {
+      });
-      assert.deepEqual(status.metadata.metadata, ['yes']);
+      call.on('status', function(status) {
-      done();
+        assert.deepEqual(status.metadata.metadata, ['yes']);
+        done();
+      });
     });
-  });
+    it('should be present when a client stream call succeeds', function(done) {
-  it('should be present when a client stream call fails', function(done) {
+      var call = client.clientStream(function(err, data) {
-    var call = client.clientStream(function(err, data) {
+        assert.ifError(err);
-      assert(err);
+      });
+      call.write({error: false});
+      call.write({error: false});
+      call.end();
+      call.on('status', function(status) {
+        assert.deepEqual(status.metadata.metadata, ['yes']);
+        done();
+      });
     });
-    call.write({error: false});
+    it('should be present when a client stream call fails', function(done) {
-    call.write({error: true});
+      var call = client.clientStream(function(err, data) {
-    call.end();
+        assert(err);
-    call.on('status', function(status) {
+      });
-      assert.deepEqual(status.metadata.metadata, ['yes']);
+      call.write({error: false});
-      done();
+      call.write({error: true});
+      call.end();
+      call.on('status', function(status) {
+        assert.deepEqual(status.metadata.metadata, ['yes']);
+        done();
+      });
     });
-  });
+    it('should be present when a server stream call succeeds', function(done) {
-  it('should be present when a server stream call succeeds', function(done) {
+      var call = client.serverStream({error: false});
-    var call = client.serverStream({error: false});
+      call.on('data', function(){});
-    call.on('data', function(){});
+      call.on('status', function(status) {
-    call.on('status', function(status) {
+        assert.strictEqual(status.code, grpc.status.OK);
-      assert.strictEqual(status.code, grpc.status.OK);
+        assert.deepEqual(status.metadata.metadata, ['yes']);
-      assert.deepEqual(status.metadata.metadata, ['yes']);
+        done();
-      done();
+      });
     });
-  });
+    it('should be present when a server stream call fails', function(done) {
-  it('should be present when a server stream call fails', function(done) {
+      var call = client.serverStream({error: true});
-    var call = client.serverStream({error: true});
+      call.on('data', function(){});
-    call.on('data', function(){});
+      call.on('error', function(error) {
-    call.on('error', function(error) {
+        assert.deepEqual(error.metadata.metadata, ['yes']);
-      assert.deepEqual(error.metadata.metadata, ['yes']);
+        done();
-      done();
+      });
     });
-  });
+    it('should be present when a bidi stream succeeds', function(done) {
-  it('should be present when a bidi stream succeeds', function(done) {
+      var call = client.bidiStream();
-    var call = client.bidiStream();
+      call.write({error: false});
-    call.write({error: false});
+      call.write({error: false});
-    call.write({error: false});
+      call.end();
-    call.end();
+      call.on('data', function(){});
-    call.on('data', function(){});
+      call.on('status', function(status) {
-    call.on('status', function(status) {
+        assert.strictEqual(status.code, grpc.status.OK);
-      assert.strictEqual(status.code, grpc.status.OK);
+        assert.deepEqual(status.metadata.metadata, ['yes']);
-      assert.deepEqual(status.metadata.metadata, ['yes']);
+        done();
-      done();
+      });
     });
-  });
+    it('should be present when a bidi stream fails', function(done) {
-  it('should be present when a bidi stream fails', function(done) {
+      var call = client.bidiStream();
-    var call = client.bidiStream();
+      call.write({error: false});
-    call.write({error: false});
+      call.write({error: true});
-    call.write({error: true});
+      call.end();
-    call.end();
+      call.on('data', function(){});
-    call.on('data', function(){});
+      call.on('error', function(error) {
-    call.on('error', function(error) {
+        assert.deepEqual(error.metadata.metadata, ['yes']);
-      assert.deepEqual(error.metadata.metadata, ['yes']);
+        done();
-      done();
+      });
     });
   });
 });

src/ruby/ext/grpc/extconf.rb

@@ -34,13 +34,25 @@ INCLUDEDIR = RbConfig::CONFIG['includedir']
 
 if ENV.key? 'GRPC_ROOT'
   GRPC_ROOT = ENV['GRPC_ROOT']
-  if ENV.key? 'GRPC_LIB_DIR'
+else
-    GRPC_LIB_DIR = ENV['GRPC_LIB_DIR']
+  grpc_root = File.expand_path(File.join(File.dirname(__FILE__), '../../../..'))
+  if File.exist?(File.join(grpc_root, 'include/grpc/grpc.h'))
+    GRPC_ROOT = grpc_root
   else
-    GRPC_LIB_DIR = 'libs/opt'
+    GRPC_ROOT = nil
   end
+end
+
+if ENV.key? 'CONFIG'
+  GRPC_CONFIG = ENV['CONFIG']
 else
-  GRPC_ROOT = nil
+  GRPC_CONFIG = 'opt'
+end
+
+if (ENV.key? 'GRPC_LIB_DIR') && (!GRPC_ROOT.nil?)
+  GRPC_LIB_DIR = File.join(GRPC_ROOT, ENV['GRPC_LIB_DIR'])
+else
+  GRPC_LIB_DIR = File.join(File.join(GRPC_ROOT, 'libs'), GRPC_CONFIG)
 end
 
 HEADER_DIRS = [
@@ -67,7 +79,10 @@ LIB_DIRS = [
 
 unless GRPC_ROOT.nil?
   HEADER_DIRS.unshift File.join(GRPC_ROOT, 'include')
-  LIB_DIRS.unshift File.join(GRPC_ROOT, GRPC_LIB_DIR)
+  LIB_DIRS.unshift GRPC_LIB_DIR
+  unless File.exist?(File.join(GRPC_LIB_DIR, 'libgrpc.a'))
+    system("make -C #{GRPC_ROOT} static_c CONFIG=#{GRPC_CONFIG}")
+  end
 end
 
 def crash(msg)

src/ruby/ext/grpc/rb_byte_buffer.c

@@ -33,7 +33,7 @@
 
 #include "rb_byte_buffer.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
 
 #include <grpc/grpc.h>
 #include <grpc/support/slice.h>

src/ruby/ext/grpc/rb_byte_buffer.h

@@ -34,8 +34,9 @@
 #ifndef GRPC_RB_BYTE_BUFFER_H_
 #define GRPC_RB_BYTE_BUFFER_H_
 
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
-#include <ruby.h>
 
 /* Converts a char* with a length to a grpc_byte_buffer */
 grpc_byte_buffer *grpc_rb_s_to_byte_buffer(char *string, size_t length);

src/ruby/ext/grpc/rb_call.c

@@ -33,7 +33,7 @@
 
 #include "rb_call.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
 
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>

src/ruby/ext/grpc/rb_call.h

@@ -34,8 +34,9 @@
 #ifndef GRPC_RB_CALL_H_
 #define GRPC_RB_CALL_H_
 
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
-#include <ruby.h>
 
 /* Gets the wrapped call from a VALUE. */
 grpc_call* grpc_rb_get_wrapped_call(VALUE v);

src/ruby/ext/grpc/rb_channel.c

@@ -33,7 +33,7 @@
 
 #include "rb_channel.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>

src/ruby/ext/grpc/rb_channel.h

@@ -34,7 +34,8 @@
 #ifndef GRPC_RB_CHANNEL_H_
 #define GRPC_RB_CHANNEL_H_
 
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
 
 /* Initializes the Channel class. */

src/ruby/ext/grpc/rb_channel_args.c

@@ -33,7 +33,8 @@
 
 #include "rb_channel_args.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
 
 #include "rb_grpc.h"

src/ruby/ext/grpc/rb_channel_args.h

@@ -34,7 +34,8 @@
 #ifndef GRPC_RB_CHANNEL_ARGS_H_
 #define GRPC_RB_CHANNEL_ARGS_H_
 
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
 
 /* Converts a hash object containing channel args to a channel args instance.

src/ruby/ext/grpc/rb_completion_queue.h

@@ -34,8 +34,9 @@
 #ifndef GRPC_RB_COMPLETION_QUEUE_H_
 #define GRPC_RB_COMPLETION_QUEUE_H_
 
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
-#include <ruby.h>
 
 /* Gets the wrapped completion queue from the ruby wrapper */
 grpc_completion_queue *grpc_rb_get_wrapped_completion_queue(VALUE v);

src/ruby/ext/grpc/rb_credentials.c

@@ -33,7 +33,7 @@
 
 #include "rb_credentials.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>

src/ruby/ext/grpc/rb_credentials.h

@@ -34,7 +34,8 @@
 #ifndef GRPC_RB_CREDENTIALS_H_
 #define GRPC_RB_CREDENTIALS_H_
 
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/grpc_security.h>
 
 /* Initializes the ruby Credentials class. */

src/ruby/ext/grpc/rb_grpc.h

@@ -35,7 +35,8 @@
 #define GRPC_RB_H_
 
 #include <sys/time.h>
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/support/time.h>
 
 /* grpc_rb_mGrpcCore is the module containing the ruby wrapper GRPC classes. */

src/ruby/ext/grpc/rb_server.c

@@ -33,7 +33,7 @@
 
 #include "rb_server.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>

src/ruby/ext/grpc/rb_server.h

@@ -34,7 +34,8 @@
 #ifndef GRPC_RB_SERVER_H_
 #define GRPC_RB_SERVER_H_
 
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/grpc.h>
 
 /* Initializes the Server class. */

src/ruby/ext/grpc/rb_server_credentials.c

@@ -33,7 +33,7 @@
 
 #include "rb_server_credentials.h"
 
-#include <ruby.h>
+#include <ruby/ruby.h>
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>

src/ruby/ext/grpc/rb_server_credentials.h

@@ -34,7 +34,8 @@
 #ifndef GRPC_RB_SERVER_CREDENTIALS_H_
 #define GRPC_RB_SERVER_CREDENTIALS_H_
 
-#include <ruby.h>
+#include <ruby/ruby.h>
+
 #include <grpc/grpc_security.h>
 
 /* Initializes the ruby ServerCredentials class. */

third_party/protobuf

@@ -1 +1 @@
-Subproject commit 7d5cf8d7a1bd24acce56296747731051ebe1b180
+Subproject commit a8b38c598d7f65b281a72809b28117afdb760931

tools/dockerfile/grpc_node/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+cp -R /var/local/git-clone/grpc /var/local/git
+
+make clean -C /var/local/git/grpc
+
+make install_c -j12 -C /var/local/git/grpc
+
+cd /var/local/git/grpc/src/node && npm install && node-gyp rebuild

tools/dockerfile/grpc_ruby/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+cp -R /var/local/git-clone/grpc /var/local/git
+
+make clean -C /var/local/git/grpc
+
+make install_c -j12 -C /var/local/git/grpc
+
+/bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && gem update bundler && bundle && rake'

tools/gce_setup/grpc_docker.sh

@@ -428,6 +428,7 @@ grpc_interop_test_args() {
       python)       grpc_port=8050 ;;
       ruby)         grpc_port=8060 ;;
       csharp_mono)  grpc_port=8070 ;;
+      csharp_dotnet) grpc_port=8070 ;;
       *) echo "bad server_type: $1" 1>&2; return 1 ;;
     esac
     shift
@@ -870,6 +871,23 @@ grpc_launch_servers() {
   done
 }
 
+# Launch servers on windows.
+grpc_launch_windows_servers() {
+   local host='jtattermusch-interop-windows2'
+   local killcmd="ps -e | grep Grpc.IntegrationTesting | awk '{print \\\$1}' | xargs kill -9"
+   echo "killing all servers and clients on $host with command $killcmd"
+   gcloud compute $project_opt ssh $zone_opt stoked-keyword-656@grpc-windows-proxy --command "ssh $host \"$killcmd\""
+
+   local cmd='cd /cygdrive/c/github/grpc/src/csharp/Grpc.IntegrationTesting.Server/bin/Debug && ./Grpc.IntegrationTesting.Server.exe --use_tls=true --port=8070'
+   # gcloud's auto-uploading of RSA keys doesn't work for Windows VMs.
+   # So we have a linux machine that is authorized to access the Windows
+   # machine through ssh and we use gcloud auth support to logon to the proxy.
+   echo "will run:"
+   echo "  $cmd"
+   echo "on $host (through grpc-windows-proxy)"
+   gcloud compute $project_opt ssh $zone_opt stoked-keyword-656@grpc-windows-proxy --command "ssh $host '$cmd'"
+}
+
 # Runs a test command on a docker instance
 #
 # The test command is issued via gcloud compute
@@ -949,6 +967,7 @@ test_runner() {
 #   node:   8040
 #   python: 8050
 #   ruby:   8060
+#   csharp: 8070
 #
 # each client_type should have an associated bash func:
 #   grpc_interop_gen_<client_type>_cmd

tools/run_tests/build_ruby.sh

@@ -31,17 +31,10 @@
 
 set -ex
 
-CONFIG=${CONFIG:-opt}
+export CONFIG=${CONFIG:-opt}
 
-# change to grpc repo root
+# change to grpc's ruby directory
-cd $(dirname $0)/../..
+cd $(dirname $0)/../../src/ruby
-
-# tells npm install to look for files in that directory
-export GRPC_ROOT=`pwd`
-# tells npm install the subdirectory with library files
-export GRPC_LIB_SUBDIR=libs/$CONFIG
-
-cd src/ruby
 
 bundle install
 rake compile:grpc

tools/run_tests/run_sanity.sh

@@ -45,6 +45,6 @@ git submodule > $submodules
 diff -u $submodules - << EOF
  05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f)
  3df69d3aefde7671053d4e3c242b228e5d79c83f third_party/openssl (OpenSSL_1_0_2a)
- 7d5cf8d7a1bd24acce56296747731051ebe1b180 third_party/protobuf (v3.0.0-alpha-1-150-g7d5cf8d)
+ a8b38c598d7f65b281a72809b28117afdb760931 third_party/protobuf (v3.0.0-alpha-1-978-ga8b38c5)
  50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8)
 EOF

tools/run_tests/run_tests.py

@@ -214,7 +214,7 @@ class RubyLanguage(object):
     return [config.job_spec(['tools/run_tests/run_ruby.sh'], None)]
 
   def make_targets(self):
-    return ['static_c']
+    return ['run_dep_checks']
 
   def build_steps(self):
     return [['tools/run_tests/build_ruby.sh']]