Merge github.com:grpc/grpc into copyright

build.json

@@ -398,8 +398,6 @@
       "build": "all",
       "language": "c++",
       "public_headers": [
-        "include/grpc++/async_server.h",
-        "include/grpc++/async_server_context.h",
         "include/grpc++/channel_arguments.h",
         "include/grpc++/channel_interface.h",
         "include/grpc++/client_context.h",
@@ -407,6 +405,8 @@
         "include/grpc++/config.h",
         "include/grpc++/create_channel.h",
         "include/grpc++/credentials.h",
+        "include/grpc++/impl/call.h",
+        "include/grpc++/impl/client_unary_call.h",
         "include/grpc++/impl/internal_stub.h",
         "include/grpc++/impl/rpc_method.h",
         "include/grpc++/impl/rpc_service_method.h",
@@ -421,30 +421,26 @@
       "headers": [
         "src/cpp/client/channel.h",
         "src/cpp/proto/proto_utils.h",
-        "src/cpp/server/server_rpc_handler.h",
         "src/cpp/server/thread_pool.h",
-        "src/cpp/stream/stream_context.h",
         "src/cpp/util/time.h"
       ],
       "src": [
         "src/cpp/client/channel.cc",
         "src/cpp/client/channel_arguments.cc",
         "src/cpp/client/client_context.cc",
+        "src/cpp/client/client_unary_call.cc",
         "src/cpp/client/create_channel.cc",
         "src/cpp/client/credentials.cc",
         "src/cpp/client/internal_stub.cc",
+        "src/cpp/common/call.cc",
         "src/cpp/common/completion_queue.cc",
         "src/cpp/common/rpc_method.cc",
         "src/cpp/proto/proto_utils.cc",
-        "src/cpp/server/async_server.cc",
-        "src/cpp/server/async_server_context.cc",
         "src/cpp/server/server.cc",
         "src/cpp/server/server_builder.cc",
-        "src/cpp/server/server_context_impl.cc",
+        "src/cpp/server/server_context.cc",
         "src/cpp/server/server_credentials.cc",
-        "src/cpp/server/server_rpc_handler.cc",
         "src/cpp/server/thread_pool.cc",
-        "src/cpp/stream/stream_context.cc",
         "src/cpp/util/status.cc",
         "src/cpp/util/time.cc"
       ],
@@ -462,7 +458,6 @@
         "test/cpp/util/messages.proto",
         "test/cpp/util/echo.proto",
         "test/cpp/util/echo_duplicate.proto",
-        "test/cpp/end2end/async_test_server.cc",
         "test/cpp/util/create_test_channel.cc"
       ]
     },
@@ -1550,6 +1545,22 @@
         "gpr"
       ]
     },
+    {
+      "name": "async_end2end_test",
+      "build": "test",
+      "language": "c++",
+      "src": [
+        "test/cpp/end2end/async_end2end_test.cc"
+      ],
+      "deps": [
+        "grpc++_test_util",
+        "grpc_test_util",
+        "grpc++",
+        "grpc",
+        "gpr_test_util",
+        "gpr"
+      ]
+    },
     {
       "name": "channel_arguments_test",
       "build": "test",
@@ -1739,12 +1750,6 @@
       "name": "ruby_plugin",
       "build": "protoc",
       "language": "c++",
-      "headers": [
-        "src/compiler/cpp_generator.h",
-        "src/compiler/cpp_generator_helpers-inl.h",
-        "src/compiler/cpp_generator_map-inl.h",
-        "src/compiler/cpp_generator_string-inl.h"
-      ],
       "src": [
         "src/compiler/ruby_generator.cc",
         "src/compiler/ruby_plugin.cc"
@@ -1767,22 +1772,6 @@
         "gpr"
       ]
     },
-    {
-      "name": "sync_client_async_server_test",
-      "build": "test",
-      "language": "c++",
-      "src": [
-        "test/cpp/end2end/sync_client_async_server_test.cc"
-      ],
-      "deps": [
-        "grpc++_test_util",
-        "grpc_test_util",
-        "grpc++",
-        "grpc",
-        "gpr_test_util",
-        "gpr"
-      ]
-    },
     {
       "name": "thread_pool_test",
       "build": "test",

examples/pubsub/publisher_test.cc

@@ -107,7 +107,7 @@ class PublisherTest : public ::testing::Test {
     server_address_ << "localhost:" << port;
     ServerBuilder builder;
     builder.AddPort(server_address_.str());
-    builder.RegisterService(service_.service());
+    builder.RegisterService(&service_);
     server_ = builder.BuildAndStart();
 
     channel_ = CreateChannel(server_address_.str(), ChannelArguments());

examples/pubsub/subscriber_test.cc

@@ -106,7 +106,7 @@ class SubscriberTest : public ::testing::Test {
     server_address_ << "localhost:" << port;
     ServerBuilder builder;
     builder.AddPort(server_address_.str());
-    builder.RegisterService(service_.service());
+    builder.RegisterService(&service_);
     server_ = builder.BuildAndStart();
 
     channel_ = CreateChannel(server_address_.str(), ChannelArguments());

include/grpc++/async_server.h

@@ -1,70 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef __GRPCPP_ASYNC_SERVER_H__
-#define __GRPCPP_ASYNC_SERVER_H__
-
-#include <mutex>
-
-#include <grpc++/config.h>
-
-struct grpc_server;
-
-namespace grpc {
-class CompletionQueue;
-
-class AsyncServer {
- public:
-  explicit AsyncServer(CompletionQueue* cc);
-  ~AsyncServer();
-
-  void AddPort(const grpc::string& addr);
-
-  void Start();
-
-  // The user has to call this to get one new rpc on the completion
-  // queue.
-  void RequestOneRpc();
-
-  void Shutdown();
-
- private:
-  bool started_;
-  std::mutex shutdown_mu_;
-  bool shutdown_;
-  grpc_server* server_;
-};
-
-}  // namespace grpc
-
-#endif  // __GRPCPP_ASYNC_SERVER_H__

include/grpc++/async_server_context.h

@@ -1,95 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef __GRPCPP_ASYNC_SERVER_CONTEXT_H__
-#define __GRPCPP_ASYNC_SERVER_CONTEXT_H__
-
-#include <chrono>
-
-#include <grpc++/config.h>
-
-struct grpc_byte_buffer;
-struct grpc_call;
-struct grpc_completion_queue;
-
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
-
-using std::chrono::system_clock;
-
-namespace grpc {
-class Status;
-
-// TODO(rocking): wrap grpc c structures.
-class AsyncServerContext {
- public:
-  AsyncServerContext(grpc_call* call, const grpc::string& method,
-                     const grpc::string& host,
-                     system_clock::time_point absolute_deadline);
-  ~AsyncServerContext();
-
-  // Accept this rpc, bind it to a completion queue.
-  void Accept(grpc_completion_queue* cq);
-
-  // Read and write calls, all async. Return true for success.
-  bool StartRead(google::protobuf::Message* request);
-  bool StartWrite(const google::protobuf::Message& response, int flags);
-  bool StartWriteStatus(const Status& status);
-
-  bool ParseRead(grpc_byte_buffer* read_buffer);
-
-  grpc::string method() const { return method_; }
-  grpc::string host() const { return host_; }
-  system_clock::time_point absolute_deadline() { return absolute_deadline_; }
-
-  grpc_call* call() { return call_; }
-
- private:
-  AsyncServerContext(const AsyncServerContext&);
-  AsyncServerContext& operator=(const AsyncServerContext&);
-
-  // These properties may be moved to a ServerContext class.
-  const grpc::string method_;
-  const grpc::string host_;
-  system_clock::time_point absolute_deadline_;
-
-  google::protobuf::Message* request_;  // not owned
-  grpc_call* call_;                     // owned
-};
-
-}  // namespace grpc
-
-#endif  // __GRPCPP_ASYNC_SERVER_CONTEXT_H__

include/grpc++/channel_interface.h

@@ -35,32 +35,30 @@
 #define __GRPCPP_CHANNEL_INTERFACE_H__
 
 #include <grpc++/status.h>
+#include <grpc++/impl/call.h>
 
 namespace google {
 namespace protobuf {
 class Message;
-}
+}  // namespace protobuf
-}
+}  // namespace google
 
-namespace grpc {
+struct grpc_call;
 
+namespace grpc {
+class Call;
+class CallOpBuffer;
 class ClientContext;
+class CompletionQueue;
 class RpcMethod;
-class StreamContextInterface;
+class CallInterface;
 
-class ChannelInterface {
+class ChannelInterface : public CallHook {
  public:
   virtual ~ChannelInterface() {}
 
-  virtual Status StartBlockingRpc(const RpcMethod& method,
+  virtual Call CreateCall(const RpcMethod &method, ClientContext *context,
-                                  ClientContext* context,
+                          CompletionQueue *cq) = 0;
-                                  const google::protobuf::Message& request,
-                                  google::protobuf::Message* result) = 0;
-
-  virtual StreamContextInterface* CreateStream(
-      const RpcMethod& method, ClientContext* context,
-      const google::protobuf::Message* request,
-      google::protobuf::Message* result) = 0;
 };
 
 }  // namespace grpc

include/grpc++/client_context.h

@@ -35,8 +35,8 @@
 #define __GRPCPP_CLIENT_CONTEXT_H__
 
 #include <chrono>
+#include <map>
 #include <string>
-#include <vector>
 
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
@@ -47,8 +47,32 @@ using std::chrono::system_clock;
 struct grpc_call;
 struct grpc_completion_queue;
 
+namespace google {
+namespace protobuf {
+class Message;
+}  // namespace protobuf
+}  // namespace google
+
 namespace grpc {
 
+class CallOpBuffer;
+class ChannelInterface;
+class CompletionQueue;
+class RpcMethod;
+class Status;
+template <class R>
+class ClientReader;
+template <class W>
+class ClientWriter;
+template <class R, class W>
+class ClientReaderWriter;
+template <class R>
+class ClientAsyncReader;
+template <class W>
+class ClientAsyncWriter;
+template <class R, class W>
+class ClientAsyncReaderWriter;
+
 class ClientContext {
  public:
   ClientContext();
@@ -57,18 +81,44 @@ class ClientContext {
   void AddMetadata(const grpc::string &meta_key,
                    const grpc::string &meta_value);
 
+  std::multimap<grpc::string, grpc::string> GetServerInitialMetadata() {
+    GPR_ASSERT(initial_metadata_received_);
+    return recv_initial_metadata_;
+  }
+
+  std::multimap<grpc::string, grpc::string> GetServerTrailingMetadata() {
+    // TODO(yangg) check finished
+    return trailing_metadata_;
+  }
+
   void set_absolute_deadline(const system_clock::time_point &deadline);
   system_clock::time_point absolute_deadline();
 
-  void StartCancel();
+  void set_authority(const grpc::string& authority) {
+    authority_ = authority;
+  }
+
+  void TryCancel();
 
  private:
   // Disallow copy and assign.
   ClientContext(const ClientContext &);
   ClientContext &operator=(const ClientContext &);
 
+  friend class CallOpBuffer;
   friend class Channel;
-  friend class StreamContext;
+  template <class R>
+  friend class ::grpc::ClientReader;
+  template <class W>
+  friend class ::grpc::ClientWriter;
+  template <class R, class W>
+  friend class ::grpc::ClientReaderWriter;
+  template <class R>
+  friend class ::grpc::ClientAsyncReader;
+  template <class W>
+  friend class ::grpc::ClientAsyncWriter;
+  template <class R, class W>
+  friend class ::grpc::ClientAsyncReaderWriter;
 
   grpc_call *call() { return call_; }
   void set_call(grpc_call *call) {
@@ -81,10 +131,18 @@ class ClientContext {
 
   gpr_timespec RawDeadline() { return absolute_deadline_; }
 
+  grpc::string authority() {
+    return authority_;
+  }
+
+  bool initial_metadata_received_ = false;
   grpc_call *call_;
   grpc_completion_queue *cq_;
   gpr_timespec absolute_deadline_;
-  std::vector<std::pair<grpc::string, grpc::string> > metadata_;
+  grpc::string authority_;
+  std::multimap<grpc::string, grpc::string> send_initial_metadata_;
+  std::multimap<grpc::string, grpc::string> recv_initial_metadata_;
+  std::multimap<grpc::string, grpc::string> trailing_metadata_;
 };
 
 }  // namespace grpc

include/grpc++/completion_queue.h

@@ -34,52 +34,82 @@
 #ifndef __GRPCPP_COMPLETION_QUEUE_H__
 #define __GRPCPP_COMPLETION_QUEUE_H__
 
+#include <grpc++/impl/client_unary_call.h>
+
 struct grpc_completion_queue;
 
 namespace grpc {
 
+template <class R>
+class ClientReader;
+template <class W>
+class ClientWriter;
+template <class R, class W>
+class ClientReaderWriter;
+template <class R>
+class ServerReader;
+template <class W>
+class ServerWriter;
+template <class R, class W>
+class ServerReaderWriter;
+
+class CompletionQueue;
+class Server;
+
+class CompletionQueueTag {
+ public:
+  virtual ~CompletionQueueTag() {}
+  // Called prior to returning from Next(), return value
+  // is the status of the operation (return status is the default thing
+  // to do)
+  virtual void FinalizeResult(void **tag, bool *status) = 0;
+};
+
 // grpc_completion_queue wrapper class
 class CompletionQueue {
  public:
   CompletionQueue();
+  explicit CompletionQueue(grpc_completion_queue *take);
   ~CompletionQueue();
 
-  enum CompletionType {
-    QUEUE_CLOSED = 0,       // Shutting down.
-    RPC_END = 1,            // An RPC finished. Either at client or server.
-    CLIENT_READ_OK = 2,     // A client-side read has finished successfully.
-    CLIENT_READ_ERROR = 3,  // A client-side read has finished with error.
-    CLIENT_WRITE_OK = 4,
-    CLIENT_WRITE_ERROR = 5,
-    SERVER_RPC_NEW = 6,     // A new RPC just arrived at the server.
-    SERVER_READ_OK = 7,     // A server-side read has finished successfully.
-    SERVER_READ_ERROR = 8,  // A server-side read has finished with error.
-    SERVER_WRITE_OK = 9,
-    SERVER_WRITE_ERROR = 10,
-    // Client or server has sent half close successfully.
-    HALFCLOSE_OK = 11,
-    // New CompletionTypes may be added in the future, so user code should
-    // always
-    // handle the default case of a CompletionType that appears after such code
-    // was
-    // written.
-    DO_NOT_USE = 20,
-  };
-
   // Blocking read from queue.
-  // For QUEUE_CLOSED, *tag is not changed.
+  // Returns true if an event was received, false if the queue is ready
-  // For SERVER_RPC_NEW, *tag will be a newly allocated AsyncServerContext.
+  // for destruction.
-  // For others, *tag will be the AsyncServerContext of this rpc.
+  bool Next(void **tag, bool *ok);
-  CompletionType Next(void** tag);
 
   // Shutdown has to be called, and the CompletionQueue can only be
-  // destructed when the QUEUE_CLOSED message has been read with Next().
+  // destructed when false is returned from Next().
   void Shutdown();
 
-  grpc_completion_queue* cq() { return cq_; }
+  grpc_completion_queue *cq() { return cq_; }
 
  private:
-  grpc_completion_queue* cq_;  // owned
+  // Friend synchronous wrappers so that they can access Pluck(), which is
+  // a semi-private API geared towards the synchronous implementation.
+  template <class R>
+  friend class ::grpc::ClientReader;
+  template <class W>
+  friend class ::grpc::ClientWriter;
+  template <class R, class W>
+  friend class ::grpc::ClientReaderWriter;
+  template <class R>
+  friend class ::grpc::ServerReader;
+  template <class W>
+  friend class ::grpc::ServerWriter;
+  template <class R, class W>
+  friend class ::grpc::ServerReaderWriter;
+  friend class ::grpc::Server;
+  friend Status BlockingUnaryCall(ChannelInterface *channel,
+                                  const RpcMethod &method,
+                                  ClientContext *context,
+                                  const google::protobuf::Message &request,
+                                  google::protobuf::Message *result);
+
+  // Wraps grpc_completion_queue_pluck.
+  // Cannot be mixed with calls to Next().
+  bool Pluck(CompletionQueueTag *tag);
+
+  grpc_completion_queue *cq_;  // owned
 };
 
 }  // namespace grpc

include/grpc++/config.h

@@ -39,6 +39,7 @@
 namespace grpc {
 
 typedef std::string string;
-}
+
+}  // namespace grpc
 
 #endif  // __GRPCPP_CONFIG_H__

include/grpc++/impl/call.h

@@ -0,0 +1,145 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __GRPCPP_CALL_H__
+#define __GRPCPP_CALL_H__
+
+#include <grpc/grpc.h>
+#include <grpc++/status.h>
+#include <grpc++/completion_queue.h>
+
+#include <memory>
+#include <map>
+
+namespace google {
+namespace protobuf {
+class Message;
+}  // namespace protobuf
+}  // namespace google
+
+struct grpc_call;
+struct grpc_op;
+
+namespace grpc {
+
+class Call;
+
+class CallOpBuffer : public CompletionQueueTag {
+ public:
+  CallOpBuffer() : return_tag_(this) {}
+  ~CallOpBuffer();
+
+  void Reset(void *next_return_tag);
+
+  // Does not take ownership.
+  void AddSendInitialMetadata(
+      std::multimap<grpc::string, grpc::string> *metadata);
+  void AddSendInitialMetadata(ClientContext *ctx);
+  void AddRecvInitialMetadata(ClientContext* ctx);
+  void AddSendMessage(const google::protobuf::Message &message);
+  void AddRecvMessage(google::protobuf::Message *message);
+  void AddClientSendClose();
+  void AddClientRecvStatus(ClientContext *ctx, Status *status);
+  void AddServerSendStatus(std::multimap<grpc::string, grpc::string> *metadata,
+                           const Status &status);
+  void AddServerRecvClose(bool *cancelled);
+
+  // INTERNAL API:
+
+  // Convert to an array of grpc_op elements
+  void FillOps(grpc_op *ops, size_t *nops);
+
+  // Called by completion queue just prior to returning from Next() or Pluck()
+  void FinalizeResult(void **tag, bool *status) override;
+
+  bool got_message = false;
+
+ private:
+  void *return_tag_ = nullptr;
+  // Send initial metadata
+  bool send_initial_metadata_ = false;
+  size_t initial_metadata_count_ = 0;
+  grpc_metadata *initial_metadata_ = nullptr;
+  // Recv initial metadta
+  std::multimap<grpc::string, grpc::string> *recv_initial_metadata_ = nullptr;
+  grpc_metadata_array recv_initial_metadata_arr_ = {0, 0, nullptr};
+  // Send message
+  const google::protobuf::Message *send_message_ = nullptr;
+  grpc_byte_buffer *send_message_buf_ = nullptr;
+  // Recv message
+  google::protobuf::Message *recv_message_ = nullptr;
+  grpc_byte_buffer *recv_message_buf_ = nullptr;
+  // Client send close
+  bool client_send_close_ = false;
+  // Client recv status
+  std::multimap<grpc::string, grpc::string> *recv_trailing_metadata_ = nullptr;
+  Status *recv_status_ = nullptr;
+  grpc_metadata_array recv_trailing_metadata_arr_ = {0, 0, nullptr};
+  grpc_status_code status_code_ = GRPC_STATUS_OK;
+  char *status_details_ = nullptr;
+  size_t status_details_capacity_ = 0;
+  // Server send status
+  const Status *send_status_ = nullptr;
+  size_t trailing_metadata_count_ = 0;
+  grpc_metadata *trailing_metadata_ = nullptr;
+  int cancelled_buf_;
+  bool *recv_closed_ = nullptr;
+};
+
+// Channel and Server implement this to allow them to hook performing ops
+class CallHook {
+ public:
+  virtual ~CallHook() {}
+  virtual void PerformOpsOnCall(CallOpBuffer *ops, Call *call) = 0;
+};
+
+// Straightforward wrapping of the C call object
+class Call final {
+ public:
+  /* call is owned by the caller */
+  Call(grpc_call *call, CallHook *call_hook_, CompletionQueue *cq);
+
+  void PerformOps(CallOpBuffer *buffer);
+
+  grpc_call *call() { return call_; }
+  CompletionQueue *cq() { return cq_; }
+
+ private:
+  CallHook *call_hook_;
+  CompletionQueue *cq_;
+  grpc_call *call_;
+};
+
+}  // namespace grpc
+
+#endif  // __GRPCPP_CALL_INTERFACE_H__

include/grpc++/impl/client_unary_call.h

@@ -0,0 +1,66 @@
+/*
+*
+* Copyright 2014, Google Inc.
+* All rights reserved.
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are
+* met:
+*
+*     * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+*     * Redistributions in binary form must reproduce the above
+* copyright notice, this list of conditions and the following disclaimer
+* in the documentation and/or other materials provided with the
+* distribution.
+*     * Neither the name of Google Inc. nor the names of its
+* contributors may be used to endorse or promote products derived from
+* this software without specific prior written permission.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+*/
+
+#ifndef __GRPCPP_CLIENT_UNARY_CALL_H__
+#define __GRPCPP_CLIENT_UNARY_CALL_H__
+
+namespace google {
+namespace protobuf {
+class Message;
+}  // namespace protobuf
+}  // namespace google
+
+namespace grpc {
+
+class ChannelInterface;
+class ClientContext;
+class CompletionQueue;
+class RpcMethod;
+class Status;
+
+// Wrapper that begins an asynchronous unary call
+void AsyncUnaryCall(ChannelInterface *channel, const RpcMethod &method,
+                    ClientContext *context,
+                    const google::protobuf::Message &request,
+                    google::protobuf::Message *result, Status *status,
+                    CompletionQueue *cq, void *tag);
+
+// Wrapper that performs a blocking unary call
+Status BlockingUnaryCall(ChannelInterface *channel, const RpcMethod &method,
+                         ClientContext *context,
+                         const google::protobuf::Message &request,
+                         google::protobuf::Message *result);
+
+}  // namespace grpc
+
+#endif

include/grpc++/impl/rpc_method.h

@@ -37,8 +37,8 @@
 namespace google {
 namespace protobuf {
 class Message;
-}
+}  // namespace protobuf
-}
+}  // namespace google
 
 namespace grpc {
 

include/grpc++/impl/rpc_service_method.h

@@ -55,25 +55,14 @@ class MethodHandler {
  public:
   virtual ~MethodHandler() {}
   struct HandlerParameter {
-    HandlerParameter(ServerContext* context,
+    HandlerParameter(Call* c, ServerContext* context,
                      const google::protobuf::Message* req,
                      google::protobuf::Message* resp)
-        : server_context(context),
+        : call(c), server_context(context), request(req), response(resp) {}
-          request(req),
+    Call* call;
-          response(resp),
-          stream_context(nullptr) {}
-    HandlerParameter(ServerContext* context,
-                     const google::protobuf::Message* req,
-                     google::protobuf::Message* resp,
-                     StreamContextInterface* stream)
-        : server_context(context),
-          request(req),
-          response(resp),
-          stream_context(stream) {}
     ServerContext* server_context;
     const google::protobuf::Message* request;
     google::protobuf::Message* response;
-    StreamContextInterface* stream_context;
   };
   virtual Status RunHandler(const HandlerParameter& param) = 0;
 };
@@ -114,7 +103,7 @@ class ClientStreamingHandler : public MethodHandler {
       : func_(func), service_(service) {}
 
   Status RunHandler(const HandlerParameter& param) final {
-    ServerReader<RequestType> reader(param.stream_context);
+    ServerReader<RequestType> reader(param.call, param.server_context);
     return func_(service_, param.server_context, &reader,
                  dynamic_cast<ResponseType*>(param.response));
   }
@@ -136,7 +125,7 @@ class ServerStreamingHandler : public MethodHandler {
       : func_(func), service_(service) {}
 
   Status RunHandler(const HandlerParameter& param) final {
-    ServerWriter<ResponseType> writer(param.stream_context);
+    ServerWriter<ResponseType> writer(param.call, param.server_context);
     return func_(service_, param.server_context,
                  dynamic_cast<const RequestType*>(param.request), &writer);
   }
@@ -159,7 +148,8 @@ class BidiStreamingHandler : public MethodHandler {
       : func_(func), service_(service) {}
 
   Status RunHandler(const HandlerParameter& param) final {
-    ServerReaderWriter<ResponseType, RequestType> stream(param.stream_context);
+    ServerReaderWriter<ResponseType, RequestType> stream(param.call,
+                                                         param.server_context);
     return func_(service_, param.server_context, &stream);
   }
 
@@ -202,9 +192,7 @@ class RpcServiceMethod : public RpcMethod {
 class RpcService {
  public:
   // Takes ownership.
-  void AddMethod(RpcServiceMethod* method) {
+  void AddMethod(RpcServiceMethod* method) { methods_.emplace_back(method); }
-    methods_.push_back(std::unique_ptr<RpcServiceMethod>(method));
-  }
 
   RpcServiceMethod* GetMethod(int i) { return methods_[i].get(); }
   int GetMethodCount() const { return methods_.size(); }

include/grpc++/impl/service_type.h

@@ -0,0 +1,127 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __GRPCPP_IMPL_SERVICE_TYPE_H__
+#define __GRPCPP_IMPL_SERVICE_TYPE_H__
+
+namespace google {
+namespace protobuf {
+class Message;
+}  // namespace protobuf
+}  // namespace google
+
+namespace grpc {
+
+class Call;
+class RpcService;
+class Server;
+class ServerContext;
+class Status;
+
+class SynchronousService {
+ public:
+  virtual ~SynchronousService() {}
+  virtual RpcService* service() = 0;
+};
+
+class ServerAsyncStreamingInterface {
+ public:
+  virtual ~ServerAsyncStreamingInterface() {}
+
+  virtual void SendInitialMetadata(void* tag) = 0;
+
+ private:
+  friend class Server;
+  virtual void BindCall(Call* call) = 0;
+};
+
+class AsynchronousService {
+ public:
+  // this is Server, but in disguise to avoid a link dependency
+  class DispatchImpl {
+   public:
+    virtual void RequestAsyncCall(void* registered_method,
+                                  ServerContext* context,
+                                  ::google::protobuf::Message* request,
+                                  ServerAsyncStreamingInterface* stream,
+                                  CompletionQueue* cq, void* tag) = 0;
+  };
+
+  AsynchronousService(CompletionQueue* cq, const char** method_names,
+                      size_t method_count)
+      : cq_(cq), method_names_(method_names), method_count_(method_count) {}
+
+  ~AsynchronousService() { delete[] request_args_; }
+
+  CompletionQueue* completion_queue() const { return cq_; }
+
+ protected:
+  void RequestAsyncUnary(int index, ServerContext* context,
+                         ::google::protobuf::Message* request,
+                         ServerAsyncStreamingInterface* stream,
+                         CompletionQueue* cq, void* tag) {
+    dispatch_impl_->RequestAsyncCall(request_args_[index], context, request,
+                                     stream, cq, tag);
+  }
+  void RequestClientStreaming(int index, ServerContext* context,
+                              ServerAsyncStreamingInterface* stream,
+                              CompletionQueue* cq, void* tag) {
+    dispatch_impl_->RequestAsyncCall(request_args_[index], context, nullptr,
+                                     stream, cq, tag);
+  }
+  void RequestServerStreaming(int index, ServerContext* context,
+                              ::google::protobuf::Message* request,
+                              ServerAsyncStreamingInterface* stream,
+                              CompletionQueue* cq, void* tag) {
+    dispatch_impl_->RequestAsyncCall(request_args_[index], context, request,
+                                     stream, cq, tag);
+  }
+  void RequestBidiStreaming(int index, ServerContext* context,
+                            ServerAsyncStreamingInterface* stream,
+                            CompletionQueue* cq, void* tag) {
+    dispatch_impl_->RequestAsyncCall(request_args_[index], context, nullptr,
+                                     stream, cq, tag);
+  }
+
+ private:
+  friend class Server;
+  CompletionQueue* const cq_;
+  DispatchImpl* dispatch_impl_ = nullptr;
+  const char** const method_names_;
+  size_t method_count_;
+  void** request_args_ = nullptr;
+};
+
+}  // namespace grpc
+
+#endif  // __GRPCPP_IMPL_SERVICE_TYPE_H__

include/grpc++/server.h

@@ -35,12 +35,14 @@
 #define __GRPCPP_SERVER_H__
 
 #include <condition_variable>
-#include <map>
+#include <list>
 #include <memory>
 #include <mutex>
 
 #include <grpc++/completion_queue.h>
 #include <grpc++/config.h>
+#include <grpc++/impl/call.h>
+#include <grpc++/impl/service_type.h>
 #include <grpc++/status.h>
 
 struct grpc_server;
@@ -48,18 +50,19 @@ struct grpc_server;
 namespace google {
 namespace protobuf {
 class Message;
-}
+}  // namespace protobuf
-}
+}  // namespace google
 
 namespace grpc {
-class AsyncServerContext;
+class AsynchronousService;
 class RpcService;
 class RpcServiceMethod;
 class ServerCredentials;
 class ThreadPoolInterface;
 
 // Currently it only supports handling rpcs in a single thread.
-class Server {
+class Server final : private CallHook,
+                     private AsynchronousService::DispatchImpl {
  public:
   ~Server();
 
@@ -69,22 +72,34 @@ class Server {
  private:
   friend class ServerBuilder;
 
+  class SyncRequest;
+  class AsyncRequest;
+
   // ServerBuilder use only
-  Server(ThreadPoolInterface* thread_pool, ServerCredentials* creds);
+  Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
+         ServerCredentials* creds);
   Server();
   // Register a service. This call does not take ownership of the service.
   // The service must exist for the lifetime of the Server instance.
-  void RegisterService(RpcService* service);
+  bool RegisterService(RpcService* service);
+  bool RegisterAsyncService(AsynchronousService* service);
   // Add a listening port. Can be called multiple times.
-  void AddPort(const grpc::string& addr);
+  int AddPort(const grpc::string& addr);
   // Start the server.
-  void Start();
+  bool Start();
 
-  void AllowOneRpc();
   void HandleQueueClosed();
   void RunRpc();
   void ScheduleCallback();
 
+  void PerformOpsOnCall(CallOpBuffer* ops, Call* call) override;
+
+  // DispatchImpl
+  void RequestAsyncCall(void* registered_method, ServerContext* context,
+                        ::google::protobuf::Message* request,
+                        ServerAsyncStreamingInterface* stream,
+                        CompletionQueue* cq, void* tag);
+
   // Completion queue.
   CompletionQueue cq_;
 
@@ -96,12 +111,11 @@ class Server {
   int num_running_cb_;
   std::condition_variable callback_cv_;
 
+  std::list<SyncRequest> sync_methods_;
+
   // Pointer to the c grpc server.
   grpc_server* server_;
 
-  // A map for all method information.
-  std::map<grpc::string, RpcServiceMethod*> method_map_;
-
   ThreadPoolInterface* thread_pool_;
   // Whether the thread pool is created and owned by the server.
   bool thread_pool_owned_;

include/grpc++/server_builder.h

@@ -41,9 +41,12 @@
 
 namespace grpc {
 
+class AsynchronousService;
+class CompletionQueue;
 class RpcService;
 class Server;
 class ServerCredentials;
+class SynchronousService;
 class ThreadPoolInterface;
 
 class ServerBuilder {
@@ -53,7 +56,13 @@ class ServerBuilder {
   // Register a service. This call does not take ownership of the service.
   // The service must exist for the lifetime of the Server instance returned by
   // BuildAndStart().
-  void RegisterService(RpcService* service);
+  void RegisterService(SynchronousService* service);
+
+  // Register an asynchronous service. New calls will be delevered to cq.
+  // This call does not take ownership of the service or completion queue.
+  // The service and completion queuemust exist for the lifetime of the Server
+  // instance returned by BuildAndStart().
+  void RegisterAsyncService(AsynchronousService* service);
 
   // Add a listening port. Can be called multiple times.
   void AddPort(const grpc::string& addr);
@@ -71,9 +80,10 @@ class ServerBuilder {
 
  private:
   std::vector<RpcService*> services_;
+  std::vector<AsynchronousService*> async_services_;
   std::vector<grpc::string> ports_;
   std::shared_ptr<ServerCredentials> creds_;
-  ThreadPoolInterface* thread_pool_;
+  ThreadPoolInterface* thread_pool_ = nullptr;
 };
 
 }  // namespace grpc

include/grpc++/server_context.h

@@ -35,15 +35,77 @@
 #define __GRPCPP_SERVER_CONTEXT_H_
 
 #include <chrono>
+#include <map>
+
+#include "config.h"
+
+struct gpr_timespec;
+struct grpc_metadata;
+struct grpc_call;
 
 namespace grpc {
 
+template <class W, class R>
+class ServerAsyncReader;
+template <class W>
+class ServerAsyncWriter;
+template <class W>
+class ServerAsyncResponseWriter;
+template <class R, class W>
+class ServerAsyncReaderWriter;
+template <class R>
+class ServerReader;
+template <class W>
+class ServerWriter;
+template <class R, class W>
+class ServerReaderWriter;
+
+class CallOpBuffer;
+class Server;
+
 // Interface of server side rpc context.
-class ServerContext {
+class ServerContext final {
  public:
-  virtual ~ServerContext() {}
+  ServerContext();  // for async calls
+  ~ServerContext();
+
+  std::chrono::system_clock::time_point absolute_deadline() {
+    return deadline_;
+  }
+
+  void AddInitialMetadata(const grpc::string& key, const grpc::string& value);
+  void AddTrailingMetadata(const grpc::string& key, const grpc::string& value);
+
+  std::multimap<grpc::string, grpc::string> client_metadata() {
+    return client_metadata_;
+  }
+
+ private:
+  friend class ::grpc::Server;
+  template <class W, class R>
+  friend class ::grpc::ServerAsyncReader;
+  template <class W>
+  friend class ::grpc::ServerAsyncWriter;
+  template <class W>
+  friend class ::grpc::ServerAsyncResponseWriter;
+  template <class R, class W>
+  friend class ::grpc::ServerAsyncReaderWriter;
+  template <class R>
+  friend class ::grpc::ServerReader;
+  template <class W>
+  friend class ::grpc::ServerWriter;
+  template <class R, class W>
+  friend class ::grpc::ServerReaderWriter;
+
+  ServerContext(gpr_timespec deadline, grpc_metadata* metadata,
+                size_t metadata_count);
 
-  virtual std::chrono::system_clock::time_point absolute_deadline() const = 0;
+  std::chrono::system_clock::time_point deadline_;
+  grpc_call* call_ = nullptr;
+  bool sent_initial_metadata_ = false;
+  std::multimap<grpc::string, grpc::string> client_metadata_;
+  std::multimap<grpc::string, grpc::string> initial_metadata_;
+  std::multimap<grpc::string, grpc::string> trailing_metadata_;
 };
 
 }  // namespace grpc

include/grpc++/stream.h

@@ -34,7 +34,12 @@
 #ifndef __GRPCPP_STREAM_H__
 #define __GRPCPP_STREAM_H__
 
-#include <grpc++/stream_context_interface.h>
+#include <grpc++/channel_interface.h>
+#include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
+#include <grpc++/server_context.h>
+#include <grpc++/impl/call.h>
+#include <grpc++/impl/service_type.h>
 #include <grpc++/status.h>
 #include <grpc/support/log.h>
 
@@ -45,16 +50,12 @@ class ClientStreamingInterface {
  public:
   virtual ~ClientStreamingInterface() {}
 
-  // Try to cancel the stream. Wait() still needs to be called to get the final
-  // status. Cancelling after the stream has finished has no effects.
-  virtual void Cancel() = 0;
-
   // Wait until the stream finishes, and return the final status. When the
   // client side declares it has no more message to send, either implicitly or
   // by calling WritesDone, it needs to make sure there is no more message to
   // be received from the server, either implicitly or by getting a false from
   // a Read(). Otherwise, this implicitly cancels the stream.
-  virtual const Status& Wait() = 0;
+  virtual Status Finish() = 0;
 };
 
 // An interface that yields a sequence of R messages.
@@ -82,147 +83,691 @@ class WriterInterface {
 };
 
 template <class R>
-class ClientReader : public ClientStreamingInterface,
+class ClientReader final : public ClientStreamingInterface,
                            public ReaderInterface<R> {
  public:
   // Blocking create a stream and write the first request out.
-  explicit ClientReader(StreamContextInterface* context) : context_(context) {
+  ClientReader(ChannelInterface* channel, const RpcMethod& method,
-    GPR_ASSERT(context_);
+               ClientContext* context, const google::protobuf::Message& request)
-    context_->Start(true);
+      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    context_->Write(context_->request(), true);
+    CallOpBuffer buf;
+    buf.AddSendInitialMetadata(&context->send_initial_metadata_);
+    buf.AddSendMessage(request);
+    buf.AddClientSendClose();
+    call_.PerformOps(&buf);
+    cq_.Pluck(&buf);
   }
 
-  ~ClientReader() { delete context_; }
+  // Blocking wait for initial metadata from server. The received metadata
-
+  // can only be accessed after this call returns. Should only be called before
-  virtual bool Read(R* msg) { return context_->Read(msg); }
+  // the first read. Calling this method is optional, and if it is not called
+  // the metadata will be available in ClientContext after the first read.
+  void WaitForInitialMetadata() {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    CallOpBuffer buf;
+    buf.AddRecvInitialMetadata(context_);
+    call_.PerformOps(&buf);
+    GPR_ASSERT(cq_.Pluck(&buf));
+  }
 
-  virtual void Cancel() { context_->Cancel(); }
+  virtual bool Read(R* msg) override {
+    CallOpBuffer buf;
+    if (!context_->initial_metadata_received_) {
+      buf.AddRecvInitialMetadata(context_);
+    }
+    buf.AddRecvMessage(msg);
+    call_.PerformOps(&buf);
+    return cq_.Pluck(&buf) && buf.got_message;
+  }
 
-  virtual const Status& Wait() { return context_->Wait(); }
+  virtual Status Finish() override {
+    CallOpBuffer buf;
+    Status status;
+    buf.AddClientRecvStatus(context_, &status);
+    call_.PerformOps(&buf);
+    GPR_ASSERT(cq_.Pluck(&buf));
+    return status;
+  }
 
  private:
-  StreamContextInterface* const context_;
+  ClientContext* context_;
+  CompletionQueue cq_;
+  Call call_;
 };
 
 template <class W>
-class ClientWriter : public ClientStreamingInterface,
+class ClientWriter final : public ClientStreamingInterface,
                            public WriterInterface<W> {
  public:
   // Blocking create a stream.
-  explicit ClientWriter(StreamContextInterface* context) : context_(context) {
+  ClientWriter(ChannelInterface* channel, const RpcMethod& method,
-    GPR_ASSERT(context_);
+               ClientContext* context, google::protobuf::Message* response)
-    context_->Start(false);
+      : context_(context),
+        response_(response),
+        call_(channel->CreateCall(method, context, &cq_)) {
+    CallOpBuffer buf;
+    buf.AddSendInitialMetadata(&context->send_initial_metadata_);
+    call_.PerformOps(&buf);
+    cq_.Pluck(&buf);
   }
 
-  ~ClientWriter() { delete context_; }
+  virtual bool Write(const W& msg) override {
-
+    CallOpBuffer buf;
-  virtual bool Write(const W& msg) {
+    buf.AddSendMessage(msg);
-    return context_->Write(const_cast<W*>(&msg), false);
+    call_.PerformOps(&buf);
+    return cq_.Pluck(&buf);
   }
 
-  virtual void WritesDone() { context_->Write(nullptr, true); }
+  virtual bool WritesDone() {
-
+    CallOpBuffer buf;
-  virtual void Cancel() { context_->Cancel(); }
+    buf.AddClientSendClose();
+    call_.PerformOps(&buf);
+    return cq_.Pluck(&buf);
+  }
 
   // Read the final response and wait for the final status.
-  virtual const Status& Wait() {
+  virtual Status Finish() override {
-    bool success = context_->Read(context_->response());
+    CallOpBuffer buf;
-    if (!success) {
+    Status status;
-      Cancel();
+    buf.AddRecvMessage(response_);
-    } else {
+    buf.AddClientRecvStatus(context_, &status);
-      success = context_->Read(nullptr);
+    call_.PerformOps(&buf);
-      if (success) {
+    GPR_ASSERT(cq_.Pluck(&buf) && buf.got_message);
-        Cancel();
+    return status;
-      }
-    }
-    return context_->Wait();
   }
 
  private:
-  StreamContextInterface* const context_;
+  ClientContext* context_;
+  google::protobuf::Message* const response_;
+  CompletionQueue cq_;
+  Call call_;
 };
 
 // Client-side interface for bi-directional streaming.
 template <class W, class R>
-class ClientReaderWriter : public ClientStreamingInterface,
+class ClientReaderWriter final : public ClientStreamingInterface,
                                  public WriterInterface<W>,
                                  public ReaderInterface<R> {
  public:
   // Blocking create a stream.
-  explicit ClientReaderWriter(StreamContextInterface* context)
+  ClientReaderWriter(ChannelInterface* channel, const RpcMethod& method,
-      : context_(context) {
+                     ClientContext* context)
-    GPR_ASSERT(context_);
+      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    context_->Start(false);
+    CallOpBuffer buf;
+    buf.AddSendInitialMetadata(&context->send_initial_metadata_);
+    call_.PerformOps(&buf);
+    GPR_ASSERT(cq_.Pluck(&buf));
   }
 
-  ~ClientReaderWriter() { delete context_; }
+  // Blocking wait for initial metadata from server. The received metadata
-
+  // can only be accessed after this call returns. Should only be called before
-  virtual bool Read(R* msg) { return context_->Read(msg); }
+  // the first read. Calling this method is optional, and if it is not called
+  // the metadata will be available in ClientContext after the first read.
+  void WaitForInitialMetadata() {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    CallOpBuffer buf;
+    buf.AddRecvInitialMetadata(context_);
+    call_.PerformOps(&buf);
+    GPR_ASSERT(cq_.Pluck(&buf));
+  }
 
-  virtual bool Write(const W& msg) {
+  virtual bool Read(R* msg) override {
-    return context_->Write(const_cast<W*>(&msg), false);
+    CallOpBuffer buf;
+    if (!context_->initial_metadata_received_) {
+      buf.AddRecvInitialMetadata(context_);
+    }
+    buf.AddRecvMessage(msg);
+    call_.PerformOps(&buf);
+    return cq_.Pluck(&buf) && buf.got_message;
   }
 
-  virtual void WritesDone() { context_->Write(nullptr, true); }
+  virtual bool Write(const W& msg) override {
+    CallOpBuffer buf;
+    buf.AddSendMessage(msg);
+    call_.PerformOps(&buf);
+    return cq_.Pluck(&buf);
+  }
 
-  virtual void Cancel() { context_->Cancel(); }
+  virtual bool WritesDone() {
+    CallOpBuffer buf;
+    buf.AddClientSendClose();
+    call_.PerformOps(&buf);
+    return cq_.Pluck(&buf);
+  }
 
-  virtual const Status& Wait() { return context_->Wait(); }
+  virtual Status Finish() override {
+    CallOpBuffer buf;
+    Status status;
+    buf.AddClientRecvStatus(context_, &status);
+    call_.PerformOps(&buf);
+    GPR_ASSERT(cq_.Pluck(&buf));
+    return status;
+  }
 
  private:
-  StreamContextInterface* const context_;
+  ClientContext* context_;
+  CompletionQueue cq_;
+  Call call_;
 };
 
 template <class R>
-class ServerReader : public ReaderInterface<R> {
+class ServerReader final : public ReaderInterface<R> {
  public:
-  explicit ServerReader(StreamContextInterface* context) : context_(context) {
+  ServerReader(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
-    GPR_ASSERT(context_);
+
-    context_->Start(true);
+  void SendInitialMetadata() {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    CallOpBuffer buf;
+    buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_->PerformOps(&buf);
+    call_->cq()->Pluck(&buf);
   }
 
-  virtual bool Read(R* msg) { return context_->Read(msg); }
+  virtual bool Read(R* msg) override {
+    CallOpBuffer buf;
+    buf.AddRecvMessage(msg);
+    call_->PerformOps(&buf);
+    return call_->cq()->Pluck(&buf) && buf.got_message;
+  }
 
  private:
-  StreamContextInterface* const context_;  // not owned
+  Call* const call_;
+  ServerContext* const ctx_;
 };
 
 template <class W>
-class ServerWriter : public WriterInterface<W> {
+class ServerWriter final : public WriterInterface<W> {
  public:
-  explicit ServerWriter(StreamContextInterface* context) : context_(context) {
+  ServerWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
-    GPR_ASSERT(context_);
+
-    context_->Start(true);
+  void SendInitialMetadata() {
-    context_->Read(context_->request());
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    CallOpBuffer buf;
+    buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_->PerformOps(&buf);
+    call_->cq()->Pluck(&buf);
   }
 
-  virtual bool Write(const W& msg) {
+  virtual bool Write(const W& msg) override {
-    return context_->Write(const_cast<W*>(&msg), false);
+    CallOpBuffer buf;
+    if (!ctx_->sent_initial_metadata_) {
+      buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    buf.AddSendMessage(msg);
+    call_->PerformOps(&buf);
+    return call_->cq()->Pluck(&buf);
   }
 
  private:
-  StreamContextInterface* const context_;  // not owned
+  Call* const call_;
+  ServerContext* const ctx_;
 };
 
 // Server-side interface for bi-directional streaming.
 template <class W, class R>
-class ServerReaderWriter : public WriterInterface<W>,
+class ServerReaderWriter final : public WriterInterface<W>,
                                  public ReaderInterface<R> {
  public:
-  explicit ServerReaderWriter(StreamContextInterface* context)
+  ServerReaderWriter(Call* call, ServerContext* ctx) : call_(call), ctx_(ctx) {}
-      : context_(context) {
+
-    GPR_ASSERT(context_);
+  void SendInitialMetadata() {
-    context_->Start(true);
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    CallOpBuffer buf;
+    buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_->PerformOps(&buf);
+    call_->cq()->Pluck(&buf);
   }
 
-  virtual bool Read(R* msg) { return context_->Read(msg); }
+  virtual bool Read(R* msg) override {
+    CallOpBuffer buf;
+    buf.AddRecvMessage(msg);
+    call_->PerformOps(&buf);
+    return call_->cq()->Pluck(&buf) && buf.got_message;
+  }
 
-  virtual bool Write(const W& msg) {
+  virtual bool Write(const W& msg) override {
-    return context_->Write(const_cast<W*>(&msg), false);
+    CallOpBuffer buf;
+    if (!ctx_->sent_initial_metadata_) {
+      buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    buf.AddSendMessage(msg);
+    call_->PerformOps(&buf);
+    return call_->cq()->Pluck(&buf);
+  }
+
+ private:
+  Call* const call_;
+  ServerContext* const ctx_;
+};
+
+// Async interfaces
+// Common interface for all client side streaming.
+class ClientAsyncStreamingInterface {
+ public:
+  virtual ~ClientAsyncStreamingInterface() {}
+
+  virtual void ReadInitialMetadata(void* tag) = 0;
+
+  virtual void Finish(Status* status, void* tag) = 0;
+};
+
+// An interface that yields a sequence of R messages.
+template <class R>
+class AsyncReaderInterface {
+ public:
+  virtual ~AsyncReaderInterface() {}
+
+  virtual void Read(R* msg, void* tag) = 0;
+};
+
+// An interface that can be fed a sequence of W messages.
+template <class W>
+class AsyncWriterInterface {
+ public:
+  virtual ~AsyncWriterInterface() {}
+
+  virtual void Write(const W& msg, void* tag) = 0;
+};
+
+template <class R>
+class ClientAsyncReader final : public ClientAsyncStreamingInterface,
+                                public AsyncReaderInterface<R> {
+ public:
+  // Create a stream and write the first request out.
+  ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
+                    const RpcMethod& method, ClientContext* context,
+                    const google::protobuf::Message& request, void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    init_buf_.Reset(tag);
+    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
+    init_buf_.AddSendMessage(request);
+    init_buf_.AddClientSendClose();
+    call_.PerformOps(&init_buf_);
+  }
+
+  void ReadInitialMetadata(void* tag) override {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddRecvInitialMetadata(context_);
+    call_.PerformOps(&meta_buf_);
+  }
+
+  void Read(R* msg, void* tag) override {
+    read_buf_.Reset(tag);
+    if (!context_->initial_metadata_received_) {
+      read_buf_.AddRecvInitialMetadata(context_);
+    }
+    read_buf_.AddRecvMessage(msg);
+    call_.PerformOps(&read_buf_);
+  }
+
+  void Finish(Status* status, void* tag) override {
+    finish_buf_.Reset(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_buf_.AddRecvInitialMetadata(context_);
+    }
+    finish_buf_.AddClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+ private:
+  ClientContext* context_ = nullptr;
+  Call call_;
+  CallOpBuffer init_buf_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer read_buf_;
+  CallOpBuffer finish_buf_;
+};
+
+template <class W>
+class ClientAsyncWriter final : public ClientAsyncStreamingInterface,
+                                public AsyncWriterInterface<W> {
+ public:
+  ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
+                    const RpcMethod& method, ClientContext* context,
+                    google::protobuf::Message* response, void* tag)
+      : context_(context),
+        response_(response),
+        call_(channel->CreateCall(method, context, cq)) {
+    init_buf_.Reset(tag);
+    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
+    call_.PerformOps(&init_buf_);
+  }
+
+  void ReadInitialMetadata(void* tag) override {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddRecvInitialMetadata(context_);
+    call_.PerformOps(&meta_buf_);
+  }
+
+  void Write(const W& msg, void* tag) override {
+    write_buf_.Reset(tag);
+    write_buf_.AddSendMessage(msg);
+    call_.PerformOps(&write_buf_);
+  }
+
+  void WritesDone(void* tag) {
+    writes_done_buf_.Reset(tag);
+    writes_done_buf_.AddClientSendClose();
+    call_.PerformOps(&writes_done_buf_);
+  }
+
+  void Finish(Status* status, void* tag) override {
+    finish_buf_.Reset(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_buf_.AddRecvInitialMetadata(context_);
+    }
+    finish_buf_.AddRecvMessage(response_);
+    finish_buf_.AddClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+ private:
+  ClientContext* context_ = nullptr;
+  google::protobuf::Message* const response_;
+  Call call_;
+  CallOpBuffer init_buf_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer write_buf_;
+  CallOpBuffer writes_done_buf_;
+  CallOpBuffer finish_buf_;
+};
+
+// Client-side interface for bi-directional streaming.
+template <class W, class R>
+class ClientAsyncReaderWriter final : public ClientAsyncStreamingInterface,
+                                      public AsyncWriterInterface<W>,
+                                      public AsyncReaderInterface<R> {
+ public:
+  ClientAsyncReaderWriter(ChannelInterface* channel, CompletionQueue* cq,
+                          const RpcMethod& method, ClientContext* context,
+                          void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    init_buf_.Reset(tag);
+    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
+    call_.PerformOps(&init_buf_);
+  }
+
+  void ReadInitialMetadata(void* tag) override {
+    GPR_ASSERT(!context_->initial_metadata_received_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddRecvInitialMetadata(context_);
+    call_.PerformOps(&meta_buf_);
+  }
+
+  void Read(R* msg, void* tag) override {
+    read_buf_.Reset(tag);
+    if (!context_->initial_metadata_received_) {
+      read_buf_.AddRecvInitialMetadata(context_);
+    }
+    read_buf_.AddRecvMessage(msg);
+    call_.PerformOps(&read_buf_);
+  }
+
+  void Write(const W& msg, void* tag) override {
+    write_buf_.Reset(tag);
+    write_buf_.AddSendMessage(msg);
+    call_.PerformOps(&write_buf_);
+  }
+
+  void WritesDone(void* tag) {
+    writes_done_buf_.Reset(tag);
+    writes_done_buf_.AddClientSendClose();
+    call_.PerformOps(&writes_done_buf_);
+  }
+
+  void Finish(Status* status, void* tag) override {
+    finish_buf_.Reset(tag);
+    if (!context_->initial_metadata_received_) {
+      finish_buf_.AddRecvInitialMetadata(context_);
+    }
+    finish_buf_.AddClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+ private:
+  ClientContext* context_ = nullptr;
+  Call call_;
+  CallOpBuffer init_buf_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer read_buf_;
+  CallOpBuffer write_buf_;
+  CallOpBuffer writes_done_buf_;
+  CallOpBuffer finish_buf_;
+};
+
+// TODO(yangg) Move out of stream.h
+template <class W>
+class ServerAsyncResponseWriter final : public ServerAsyncStreamingInterface {
+ public:
+  explicit ServerAsyncResponseWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_buf_);
+  }
+
+  void Finish(const W& msg, const Status& status, void* tag) {
+    finish_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // The response is dropped if the status is not OK.
+    if (status.IsOk()) {
+      finish_buf_.AddSendMessage(msg);
+    }
+    bool cancelled = false;
+    finish_buf_.AddServerRecvClose(&cancelled);
+    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+  void FinishWithError(const Status& status, void* tag) {
+    GPR_ASSERT(!status.IsOk());
+    finish_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    bool cancelled = false;
+    finish_buf_.AddServerRecvClose(&cancelled);
+    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+ private:
+  void BindCall(Call* call) override { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer finish_buf_;
+};
+
+template <class W, class R>
+class ServerAsyncReader : public ServerAsyncStreamingInterface,
+                          public AsyncReaderInterface<R> {
+ public:
+  explicit ServerAsyncReader(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) override {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_buf_);
+  }
+
+  void Read(R* msg, void* tag) override {
+    read_buf_.Reset(tag);
+    read_buf_.AddRecvMessage(msg);
+    call_.PerformOps(&read_buf_);
+  }
+
+  void Finish(const W& msg, const Status& status, void* tag) {
+    finish_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    // The response is dropped if the status is not OK.
+    if (status.IsOk()) {
+      finish_buf_.AddSendMessage(msg);
+    }
+    bool cancelled = false;
+    finish_buf_.AddServerRecvClose(&cancelled);
+    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+  void FinishWithError(const Status& status, void* tag) {
+    GPR_ASSERT(!status.IsOk());
+    finish_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    bool cancelled = false;
+    finish_buf_.AddServerRecvClose(&cancelled);
+    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+ private:
+  void BindCall(Call* call) override { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer read_buf_;
+  CallOpBuffer finish_buf_;
+};
+
+template <class W>
+class ServerAsyncWriter : public ServerAsyncStreamingInterface,
+                          public AsyncWriterInterface<W> {
+ public:
+  explicit ServerAsyncWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) override {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_buf_);
+  }
+
+  void Write(const W& msg, void* tag) override {
+    write_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      write_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    write_buf_.AddSendMessage(msg);
+    call_.PerformOps(&write_buf_);
+  }
+
+  void Finish(const Status& status, void* tag) {
+    finish_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    bool cancelled = false;
+    finish_buf_.AddServerRecvClose(&cancelled);
+    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_buf_);
+  }
+
+ private:
+  void BindCall(Call* call) override { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer write_buf_;
+  CallOpBuffer finish_buf_;
+};
+
+// Server-side interface for bi-directional streaming.
+template <class W, class R>
+class ServerAsyncReaderWriter : public ServerAsyncStreamingInterface,
+                                public AsyncWriterInterface<W>,
+                                public AsyncReaderInterface<R> {
+ public:
+  explicit ServerAsyncReaderWriter(ServerContext* ctx)
+      : call_(nullptr, nullptr, nullptr), ctx_(ctx) {}
+
+  void SendInitialMetadata(void* tag) override {
+    GPR_ASSERT(!ctx_->sent_initial_metadata_);
+
+    meta_buf_.Reset(tag);
+    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    ctx_->sent_initial_metadata_ = true;
+    call_.PerformOps(&meta_buf_);
+  }
+
+  virtual void Read(R* msg, void* tag) override {
+    read_buf_.Reset(tag);
+    read_buf_.AddRecvMessage(msg);
+    call_.PerformOps(&read_buf_);
+  }
+
+  virtual void Write(const W& msg, void* tag) override {
+    write_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      write_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    write_buf_.AddSendMessage(msg);
+    call_.PerformOps(&write_buf_);
+  }
+
+  void Finish(const Status& status, void* tag) {
+    finish_buf_.Reset(tag);
+    if (!ctx_->sent_initial_metadata_) {
+      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ctx_->sent_initial_metadata_ = true;
+    }
+    bool cancelled = false;
+    finish_buf_.AddServerRecvClose(&cancelled);
+    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_buf_);
   }
 
  private:
-  StreamContextInterface* const context_;  // not owned
+  void BindCall(Call* call) override { call_ = *call; }
+
+  Call call_;
+  ServerContext* ctx_;
+  CallOpBuffer meta_buf_;
+  CallOpBuffer read_buf_;
+  CallOpBuffer write_buf_;
+  CallOpBuffer finish_buf_;
 };
 
 }  // namespace grpc

include/grpc++/stream_context_interface.h

@@ -1,64 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef __GRPCPP_STREAM_CONTEXT_INTERFACE_H__
-#define __GRPCPP_STREAM_CONTEXT_INTERFACE_H__
-
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
-
-namespace grpc {
-class Status;
-
-// An interface to avoid dependency on internal implementation.
-class StreamContextInterface {
- public:
-  virtual ~StreamContextInterface() {}
-
-  virtual void Start(bool buffered) = 0;
-
-  virtual bool Read(google::protobuf::Message* msg) = 0;
-  virtual bool Write(const google::protobuf::Message* msg, bool is_last) = 0;
-  virtual const Status& Wait() = 0;
-  virtual void Cancel() = 0;
-
-  virtual google::protobuf::Message* request() = 0;
-  virtual google::protobuf::Message* response() = 0;
-};
-
-}  // namespace grpc
-
-#endif  // __GRPCPP_STREAM_CONTEXT_INTERFACE_H__

include/grpc/grpc

@@ -0,0 +1 @@
+/home/craig/grpc-ct/include/grpc

include/grpc/grpc.h

@@ -92,7 +92,12 @@ typedef struct {
   } value;
 } grpc_arg;
 
-/* An array of arguments that can be passed around */
+/* An array of arguments that can be passed around.
+   Used to set optional channel-level configuration.
+   These configuration options are modelled as key-value pairs as defined
+   by grpc_arg; keys are strings to allow easy backwards-compatible extension
+   by arbitrary parties.
+   All evaluation is performed at channel creation time. */
 typedef struct {
   size_t num_args;
   grpc_arg *args;
@@ -255,15 +260,18 @@ void grpc_call_details_init(grpc_call_details *details);
 void grpc_call_details_destroy(grpc_call_details *details);
 
 typedef enum {
-  /* Send initial metadata: one and only one instance MUST be sent for each call,
+  /* Send initial metadata: one and only one instance MUST be sent for each
+     call,
      unless the call was cancelled - in which case this can be skipped */
   GRPC_OP_SEND_INITIAL_METADATA = 0,
   /* Send a message: 0 or more of these operations can occur for each call */
   GRPC_OP_SEND_MESSAGE,
-  /* Send a close from the server: one and only one instance MUST be sent from the client,
+  /* Send a close from the server: one and only one instance MUST be sent from
+     the client,
      unless the call was cancelled - in which case this can be skipped */
   GRPC_OP_SEND_CLOSE_FROM_CLIENT,
-  /* Send status from the server: one and only one instance MUST be sent from the server
+  /* Send status from the server: one and only one instance MUST be sent from
+     the server
      unless the call was cancelled - in which case this can be skipped */
   GRPC_OP_SEND_STATUS_FROM_SERVER,
   /* Receive initial metadata: one and only one MUST be made on the client, must
@@ -271,13 +279,16 @@ typedef enum {
   GRPC_OP_RECV_INITIAL_METADATA,
   /* Receive a message: 0 or more of these operations can occur for each call */
   GRPC_OP_RECV_MESSAGE,
-  /* Receive status on the client: one and only one must be made on the client */
+  /* Receive status on the client: one and only one must be made on the client
+     */
   GRPC_OP_RECV_STATUS_ON_CLIENT,
-  /* Receive status on the server: one and only one must be made on the server */
+  /* Receive status on the server: one and only one must be made on the server
+     */
   GRPC_OP_RECV_CLOSE_ON_SERVER
 } grpc_op_type;
 
-/* Operation data: one field for each op type (except SEND_CLOSE_FROM_CLIENT which has
+/* Operation data: one field for each op type (except SEND_CLOSE_FROM_CLIENT
+   which has
    no arguments) */
 typedef struct grpc_op {
   grpc_op_type op;
@@ -301,16 +312,20 @@ typedef struct grpc_op {
     grpc_metadata_array *recv_initial_metadata;
     grpc_byte_buffer **recv_message;
     struct {
-      /* ownership of the array is with the caller, but ownership of the elements
+      /* ownership of the array is with the caller, but ownership of the
+         elements
          stays with the call object (ie key, value members are owned by the call
          object, trailing_metadata->array is owned by the caller).
          After the operation completes, call grpc_metadata_array_destroy on this
          value, or reuse it in a future op. */
       grpc_metadata_array *trailing_metadata;
       grpc_status_code *status;
-      /* status_details is a buffer owned by the application before the op completes
+      /* status_details is a buffer owned by the application before the op
-         and after the op has completed. During the operation status_details may be
+         completes
-         reallocated to a size larger than *status_details_capacity, in which case
+         and after the op has completed. During the operation status_details may
+         be
+         reallocated to a size larger than *status_details_capacity, in which
+         case
          *status_details_capacity will be updated with the new array capacity.
 
          Pre-allocating space:
@@ -331,7 +346,8 @@ typedef struct grpc_op {
       size_t *status_details_capacity;
     } recv_status_on_client;
     struct {
-      /* out argument, set to 1 if the call failed in any way (seen as a cancellation
+      /* out argument, set to 1 if the call failed in any way (seen as a
+         cancellation
          on the server), or 0 if the call succeeded */
       int *cancelled;
     } recv_close_on_server;
@@ -400,7 +416,10 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel,
 grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
                                       size_t nops, void *tag);
 
-/* Create a client channel */
+/* Create a client channel to 'target'. Additional channel level configuration
+   MAY be provided by grpc_channel_args, though the expectation is that most
+   clients will want to simply pass NULL. See grpc_channel_args definition
+   for more on this. */
 grpc_channel *grpc_channel_create(const char *target,
                                   const grpc_channel_args *args);
 
@@ -540,12 +559,34 @@ void grpc_call_destroy(grpc_call *call);
 grpc_call_error grpc_server_request_call_old(grpc_server *server,
                                              void *tag_new);
 
+/* Request notification of a new call */
 grpc_call_error grpc_server_request_call(
     grpc_server *server, grpc_call **call, grpc_call_details *details,
     grpc_metadata_array *request_metadata,
-    grpc_completion_queue *completion_queue, void *tag_new);
+    grpc_completion_queue *cq_bound_to_call, 
+    void *tag_new);
 
-/* Create a server */
+/* Registers a method in the server.
+   Methods to this (host, method) pair will not be reported by
+   grpc_server_request_call, but instead be reported by 
+   grpc_server_request_registered_call when passed the appropriate
+   registered_method (as returned by this function).
+   Must be called before grpc_server_start.
+   Returns NULL on failure. */
+void *grpc_server_register_method(grpc_server *server, const char *method,
+                                  const char *host, 
+                                  grpc_completion_queue *new_call_cq);
+
+/* Request notification of a new pre-registered call */
+grpc_call_error grpc_server_request_registered_call(
+    grpc_server *server, void *registered_method, grpc_call **call,
+    gpr_timespec *deadline, grpc_metadata_array *request_metadata,
+    grpc_byte_buffer **optional_payload,
+    grpc_completion_queue *cq_bound_to_call, void *tag_new);
+
+/* Create a server. Additional configuration for each incoming channel can
+   be specified with args. If no additional configuration is needed, args can
+   be NULL. See grpc_channel_args for more. */
 grpc_server *grpc_server_create(grpc_completion_queue *cq,
                                 const grpc_channel_args *args);
 
@@ -559,15 +600,19 @@ void grpc_server_start(grpc_server *server);
 
 /* Begin shutting down a server.
    After completion, no new calls or connections will be admitted.
-   Existing calls will be allowed to complete. */
+   Existing calls will be allowed to complete.
+   Shutdown is idempotent. */
 void grpc_server_shutdown(grpc_server *server);
 
 /* As per grpc_server_shutdown, but send a GRPC_SERVER_SHUTDOWN event when
-   there are no more calls being serviced. */
+   there are no more calls being serviced.
+   Shutdown is idempotent, and all tags will be notified at once if multiple
+   grpc_server_shutdown_and_notify calls are made. */
 void grpc_server_shutdown_and_notify(grpc_server *server, void *tag);
 
 /* Destroy a server.
-   Forcefully cancels all existing calls. */
+   Forcefully cancels all existing calls.
+   Implies grpc_server_shutdown() if one was not previously performed. */
 void grpc_server_destroy(grpc_server *server);
 
 #ifdef __cplusplus

include/grpc/support/cpu.h

@@ -34,6 +34,10 @@
 #ifndef __GRPC_INTERNAL_SUPPORT_CPU_H__
 #define __GRPC_INTERNAL_SUPPORT_CPU_H__
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /* Interface providing CPU information for currently running system */
 
 /* Return the number of CPU cores on the current system. Will return 0 if
@@ -46,4 +50,8 @@ unsigned gpr_cpu_num_cores(void);
    [0, gpr_cpu_num_cores() - 1] */
 unsigned gpr_cpu_current_cpu(void);
 
+#ifdef __cplusplus
+}  // extern "C"
+#endif
+
 #endif /* __GRPC_INTERNAL_SUPPORT_CPU_H__ */

src/compiler/cpp_generator.cc

@@ -41,10 +41,18 @@
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <sstream>
 
 namespace grpc_cpp_generator {
 namespace {
 
+template <class T>
+std::string as_string(T x) {
+  std::ostringstream out;
+  out << x;
+  return out.str();
+}
+
 bool NoStreaming(const google::protobuf::MethodDescriptor *method) {
   return !method->client_streaming() && !method->server_streaming();
 }
@@ -61,6 +69,17 @@ bool BidiStreaming(const google::protobuf::MethodDescriptor *method) {
   return method->client_streaming() && method->server_streaming();
 }
 
+bool HasUnaryCalls(const google::protobuf::FileDescriptor *file) {
+  for (int i = 0; i < file->service_count(); i++) {
+    for (int j = 0; j < file->service(i)->method_count(); j++) {
+      if (NoStreaming(file->service(i)->method(j))) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
 bool HasClientOnlyStreaming(const google::protobuf::FileDescriptor *file) {
   for (int i = 0; i < file->service_count(); i++) {
     for (int j = 0; j < file->service(i)->method_count(); j++) {
@@ -97,20 +116,30 @@ bool HasBidiStreaming(const google::protobuf::FileDescriptor *file) {
 
 std::string GetHeaderIncludes(const google::protobuf::FileDescriptor *file) {
   std::string temp =
-      "#include \"grpc++/impl/internal_stub.h\"\n"
+      "#include <grpc++/impl/internal_stub.h>\n"
-      "#include \"grpc++/status.h\"\n"
+      "#include <grpc++/impl/service_type.h>\n"
+      "#include <grpc++/status.h>\n"
       "\n"
       "namespace grpc {\n"
+      "class CompletionQueue;\n"
       "class ChannelInterface;\n"
       "class RpcService;\n"
       "class ServerContext;\n";
+  if (HasUnaryCalls(file)) {
+    temp.append(
+        "template <class OutMessage> class ServerAsyncResponseWriter;\n");
+  }
   if (HasClientOnlyStreaming(file)) {
     temp.append("template <class OutMessage> class ClientWriter;\n");
     temp.append("template <class InMessage> class ServerReader;\n");
+    temp.append("template <class OutMessage> class ClientAsyncWriter;\n");
+    temp.append("template <class OutMessage, class InMessage> class ServerAsyncReader;\n");
   }
   if (HasServerOnlyStreaming(file)) {
     temp.append("template <class InMessage> class ClientReader;\n");
     temp.append("template <class OutMessage> class ServerWriter;\n");
+    temp.append("template <class OutMessage> class ClientAsyncReader;\n");
+    temp.append("template <class InMessage> class ServerAsyncWriter;\n");
   }
   if (HasBidiStreaming(file)) {
     temp.append(
@@ -119,16 +148,24 @@ std::string GetHeaderIncludes(const google::protobuf::FileDescriptor *file) {
     temp.append(
         "template <class OutMessage, class InMessage>\n"
         "class ServerReaderWriter;\n");
+    temp.append(
+        "template <class OutMessage, class InMessage>\n"
+        "class ClientAsyncReaderWriter;\n");
+    temp.append(
+        "template <class OutMessage, class InMessage>\n"
+        "class ServerAsyncReaderWriter;\n");
   }
   temp.append("}  // namespace grpc\n");
   return temp;
 }
 
 std::string GetSourceIncludes() {
-  return "#include \"grpc++/channel_interface.h\"\n"
+  return "#include <grpc++/channel_interface.h>\n"
-         "#include \"grpc++/impl/rpc_method.h\"\n"
+         "#include <grpc++/impl/client_unary_call.h>\n"
-         "#include \"grpc++/impl/rpc_service_method.h\"\n"
+         "#include <grpc++/impl/rpc_method.h>\n"
-         "#include \"grpc++/stream.h\"\n";
+         "#include <grpc++/impl/rpc_service_method.h>\n"
+         "#include <grpc++/impl/service_type.h>\n"
+         "#include <grpc++/stream.h>\n";
 }
 
 void PrintHeaderClientMethod(google::protobuf::io::Printer *printer,
@@ -142,25 +179,42 @@ void PrintHeaderClientMethod(google::protobuf::io::Printer *printer,
   if (NoStreaming(method)) {
     printer->Print(*vars,
                    "::grpc::Status $Method$(::grpc::ClientContext* context, "
-                   "const $Request$& request, $Response$* response);\n\n");
+                   "const $Request$& request, $Response$* response);\n");
+    printer->Print(*vars,
+                   "void $Method$(::grpc::ClientContext* context, "
+                   "const $Request$& request, $Response$* response, "
+                   "::grpc::Status* status, "
+                   "::grpc::CompletionQueue* cq, void* tag);\n");
   } else if (ClientOnlyStreaming(method)) {
-    printer->Print(
+    printer->Print(*vars,
-        *vars,
                    "::grpc::ClientWriter< $Request$>* $Method$("
-        "::grpc::ClientContext* context, $Response$* response);\n\n");
+                   "::grpc::ClientContext* context, $Response$* response);\n");
+    printer->Print(*vars,
+                   "::grpc::ClientAsyncWriter< $Request$>* $Method$("
+                   "::grpc::ClientContext* context, $Response$* response, "
+                   "::grpc::CompletionQueue* cq, void* tag);\n");
   } else if (ServerOnlyStreaming(method)) {
     printer->Print(
         *vars,
         "::grpc::ClientReader< $Response$>* $Method$("
-        "::grpc::ClientContext* context, const $Request$* request);\n\n");
+        "::grpc::ClientContext* context, const $Request$& request);\n");
+    printer->Print(*vars,
+                   "::grpc::ClientAsyncReader< $Response$>* $Method$("
+                   "::grpc::ClientContext* context, const $Request$& request, "
+                   "::grpc::CompletionQueue* cq, void* tag);\n");
   } else if (BidiStreaming(method)) {
     printer->Print(*vars,
                    "::grpc::ClientReaderWriter< $Request$, $Response$>* "
-                   "$Method$(::grpc::ClientContext* context);\n\n");
+                   "$Method$(::grpc::ClientContext* context);\n");
+    printer->Print(*vars,
+                   "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* "
+                   "$Method$(::grpc::ClientContext* context, "
+                   "::grpc::CompletionQueue* cq, void* tag);\n");
   }
 }
 
-void PrintHeaderServerMethod(google::protobuf::io::Printer *printer,
+void PrintHeaderServerMethodSync(
+    google::protobuf::io::Printer *printer,
     const google::protobuf::MethodDescriptor *method,
     std::map<std::string, std::string> *vars) {
   (*vars)["Method"] = method->name();
@@ -194,19 +248,56 @@ void PrintHeaderServerMethod(google::protobuf::io::Printer *printer,
   }
 }
 
+void PrintHeaderServerMethodAsync(
+    google::protobuf::io::Printer *printer,
+    const google::protobuf::MethodDescriptor *method,
+    std::map<std::string, std::string> *vars) {
+  (*vars)["Method"] = method->name();
+  (*vars)["Request"] =
+      grpc_cpp_generator::ClassName(method->input_type(), true);
+  (*vars)["Response"] =
+      grpc_cpp_generator::ClassName(method->output_type(), true);
+  if (NoStreaming(method)) {
+    printer->Print(*vars,
+                   "void Request$Method$("
+                   "::grpc::ServerContext* context, $Request$* request, "
+                   "::grpc::ServerAsyncResponseWriter< $Response$>* response, "
+                   "::grpc::CompletionQueue* cq, void *tag);\n");
+  } else if (ClientOnlyStreaming(method)) {
+    printer->Print(*vars,
+                   "void Request$Method$("
+                   "::grpc::ServerContext* context, "
+                   "::grpc::ServerAsyncReader< $Response$, $Request$>* reader, "
+                   "::grpc::CompletionQueue* cq, void *tag);\n");
+  } else if (ServerOnlyStreaming(method)) {
+    printer->Print(*vars,
+                   "void Request$Method$("
+                   "::grpc::ServerContext* context, $Request$* request, "
+                   "::grpc::ServerAsyncWriter< $Response$>* writer, "
+                   "::grpc::CompletionQueue* cq, void *tag);\n");
+  } else if (BidiStreaming(method)) {
+    printer->Print(
+        *vars,
+        "void Request$Method$("
+        "::grpc::ServerContext* context, "
+        "::grpc::ServerAsyncReaderWriter< $Response$, $Request$>* stream, "
+        "::grpc::CompletionQueue* cq, void *tag);\n");
+  }
+}
+
 void PrintHeaderService(google::protobuf::io::Printer *printer,
                         const google::protobuf::ServiceDescriptor *service,
                         std::map<std::string, std::string> *vars) {
   (*vars)["Service"] = service->name();
 
   printer->Print(*vars,
-                 "class $Service$ {\n"
+                 "class $Service$ final {\n"
                  " public:\n");
   printer->Indent();
 
   // Client side
   printer->Print(
-      "class Stub : public ::grpc::InternalStub {\n"
+      "class Stub final : public ::grpc::InternalStub {\n"
       " public:\n");
   printer->Indent();
   for (int i = 0; i < service->method_count(); ++i) {
@@ -220,23 +311,37 @@ void PrintHeaderService(google::protobuf::io::Printer *printer,
 
   printer->Print("\n");
 
-  // Server side
+  // Server side - Synchronous
   printer->Print(
-      "class Service {\n"
+      "class Service : public ::grpc::SynchronousService {\n"
       " public:\n");
   printer->Indent();
   printer->Print("Service() : service_(nullptr) {}\n");
   printer->Print("virtual ~Service();\n");
   for (int i = 0; i < service->method_count(); ++i) {
-    PrintHeaderServerMethod(printer, service->method(i), vars);
+    PrintHeaderServerMethodSync(printer, service->method(i), vars);
   }
-  printer->Print("::grpc::RpcService* service();\n");
+  printer->Print("::grpc::RpcService* service() override final;\n");
   printer->Outdent();
   printer->Print(
       " private:\n"
       "  ::grpc::RpcService* service_;\n");
   printer->Print("};\n");
 
+  // Server side - Asynchronous
+  printer->Print(
+      "class AsyncService final : public ::grpc::AsynchronousService {\n"
+      " public:\n");
+  printer->Indent();
+  (*vars)["MethodCount"] = as_string(service->method_count());
+  printer->Print("explicit AsyncService(::grpc::CompletionQueue* cq);\n");
+  printer->Print("~AsyncService() {};\n");
+  for (int i = 0; i < service->method_count(); ++i) {
+    PrintHeaderServerMethodAsync(printer, service->method(i), vars);
+  }
+  printer->Outdent();
+  printer->Print("};\n");
+
   printer->Outdent();
   printer->Print("};\n");
 }
@@ -268,10 +373,20 @@ void PrintSourceClientMethod(google::protobuf::io::Printer *printer,
                    "::grpc::ClientContext* context, "
                    "const $Request$& request, $Response$* response) {\n");
     printer->Print(*vars,
-                   "  return channel()->StartBlockingRpc("
+                   "  return ::grpc::BlockingUnaryCall(channel(),"
-                   "::grpc::RpcMethod(\"/$Package$$Service$/$Method$\"), "
+                   "::grpc::RpcMethod($Service$_method_names[$Idx$]), "
                    "context, request, response);\n"
                    "}\n\n");
+    printer->Print(*vars,
+                   "void $Service$::Stub::$Method$("
+                   "::grpc::ClientContext* context, "
+                   "const $Request$& request, $Response$* response, ::grpc::Status* status, "
+                   "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(*vars,
+                   "  ::grpc::AsyncUnaryCall(channel(),"
+                   "::grpc::RpcMethod($Service$_method_names[$Idx$]), "
+                   "context, request, response, status, cq, tag);\n"
+                   "}\n\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(
         *vars,
@@ -279,22 +394,46 @@ void PrintSourceClientMethod(google::protobuf::io::Printer *printer,
         "::grpc::ClientContext* context, $Response$* response) {\n");
     printer->Print(*vars,
                    "  return new ::grpc::ClientWriter< $Request$>("
-                   "channel()->CreateStream("
+                   "channel(),"
-                   "::grpc::RpcMethod(\"/$Package$$Service$/$Method$\", "
+                   "::grpc::RpcMethod($Service$_method_names[$Idx$], "
                    "::grpc::RpcMethod::RpcType::CLIENT_STREAMING), "
-                   "context, nullptr, response));\n"
+                   "context, response);\n"
+                   "}\n\n");
+    printer->Print(
+        *vars,
+        "::grpc::ClientAsyncWriter< $Request$>* $Service$::Stub::$Method$("
+        "::grpc::ClientContext* context, $Response$* response, "
+        "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(*vars,
+                   "  return new ::grpc::ClientAsyncWriter< $Request$>("
+                   "channel(), cq, "
+                   "::grpc::RpcMethod($Service$_method_names[$Idx$], "
+                   "::grpc::RpcMethod::RpcType::CLIENT_STREAMING), "
+                   "context, response, tag);\n"
                    "}\n\n");
   } else if (ServerOnlyStreaming(method)) {
     printer->Print(
         *vars,
         "::grpc::ClientReader< $Response$>* $Service$::Stub::$Method$("
-        "::grpc::ClientContext* context, const $Request$* request) {\n");
+        "::grpc::ClientContext* context, const $Request$& request) {\n");
     printer->Print(*vars,
                    "  return new ::grpc::ClientReader< $Response$>("
-                   "channel()->CreateStream("
+                   "channel(),"
-                   "::grpc::RpcMethod(\"/$Package$$Service$/$Method$\", "
+                   "::grpc::RpcMethod($Service$_method_names[$Idx$], "
                    "::grpc::RpcMethod::RpcType::SERVER_STREAMING), "
-                   "context, request, nullptr));\n"
+                   "context, request);\n"
+                   "}\n\n");
+    printer->Print(
+        *vars,
+        "::grpc::ClientAsyncReader< $Response$>* $Service$::Stub::$Method$("
+        "::grpc::ClientContext* context, const $Request$& request, "
+        "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(*vars,
+                   "  return new ::grpc::ClientAsyncReader< $Response$>("
+                   "channel(), cq, "
+                   "::grpc::RpcMethod($Service$_method_names[$Idx$], "
+                   "::grpc::RpcMethod::RpcType::SERVER_STREAMING), "
+                   "context, request, tag);\n"
                    "}\n\n");
   } else if (BidiStreaming(method)) {
     printer->Print(
@@ -304,10 +443,23 @@ void PrintSourceClientMethod(google::protobuf::io::Printer *printer,
     printer->Print(
         *vars,
         "  return new ::grpc::ClientReaderWriter< $Request$, $Response$>("
-        "channel()->CreateStream("
+        "channel(),"
-        "::grpc::RpcMethod(\"/$Package$$Service$/$Method$\", "
+        "::grpc::RpcMethod($Service$_method_names[$Idx$], "
+        "::grpc::RpcMethod::RpcType::BIDI_STREAMING), "
+        "context);\n"
+        "}\n\n");
+    printer->Print(
+        *vars,
+        "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>* "
+        "$Service$::Stub::$Method$(::grpc::ClientContext* context, "
+        "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(
+        *vars,
+        "  return new ::grpc::ClientAsyncReaderWriter< $Request$, $Response$>("
+        "channel(), cq, "
+        "::grpc::RpcMethod($Service$_method_names[$Idx$], "
         "::grpc::RpcMethod::RpcType::BIDI_STREAMING), "
-        "context, nullptr, nullptr));\n"
+        "context, tag);\n"
         "}\n\n");
   }
 }
@@ -362,10 +514,73 @@ void PrintSourceServerMethod(google::protobuf::io::Printer *printer,
   }
 }
 
+void PrintSourceServerAsyncMethod(
+    google::protobuf::io::Printer *printer,
+    const google::protobuf::MethodDescriptor *method,
+    std::map<std::string, std::string> *vars) {
+  (*vars)["Method"] = method->name();
+  (*vars)["Request"] =
+      grpc_cpp_generator::ClassName(method->input_type(), true);
+  (*vars)["Response"] =
+      grpc_cpp_generator::ClassName(method->output_type(), true);
+  if (NoStreaming(method)) {
+    printer->Print(*vars,
+                   "void $Service$::AsyncService::Request$Method$("
+                   "::grpc::ServerContext* context, "
+                   "$Request$* request, "
+                   "::grpc::ServerAsyncResponseWriter< $Response$>* response, "
+                   "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(
+        *vars,
+        "  AsynchronousService::RequestAsyncUnary($Idx$, context, request, response, cq, tag);\n");
+    printer->Print("}\n\n");
+  } else if (ClientOnlyStreaming(method)) {
+    printer->Print(*vars,
+                   "void $Service$::AsyncService::Request$Method$("
+                   "::grpc::ServerContext* context, "
+                   "::grpc::ServerAsyncReader< $Response$, $Request$>* reader, "
+                   "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(
+        *vars,
+        "  AsynchronousService::RequestClientStreaming($Idx$, context, reader, cq, tag);\n");
+    printer->Print("}\n\n");
+  } else if (ServerOnlyStreaming(method)) {
+    printer->Print(*vars,
+                   "void $Service$::AsyncService::Request$Method$("
+                   "::grpc::ServerContext* context, "
+                   "$Request$* request, "
+                   "::grpc::ServerAsyncWriter< $Response$>* writer, "
+                   "::grpc::CompletionQueue* cq, void* tag) {\n");
+    printer->Print(
+        *vars,
+        "  AsynchronousService::RequestServerStreaming($Idx$, context, request, writer, cq, tag);\n");
+    printer->Print("}\n\n");
+  } else if (BidiStreaming(method)) {
+    printer->Print(
+        *vars,
+        "void $Service$::AsyncService::Request$Method$("
+        "::grpc::ServerContext* context, "
+        "::grpc::ServerAsyncReaderWriter< $Response$, $Request$>* stream, "
+        "::grpc::CompletionQueue* cq, void *tag) {\n");
+    printer->Print(
+        *vars,
+        "  AsynchronousService::RequestBidiStreaming($Idx$, context, stream, cq, tag);\n");
+    printer->Print("}\n\n");
+  }
+}
+
 void PrintSourceService(google::protobuf::io::Printer *printer,
                         const google::protobuf::ServiceDescriptor *service,
                         std::map<std::string, std::string> *vars) {
   (*vars)["Service"] = service->name();
+
+  printer->Print(*vars, "static const char* $Service$_method_names[] = {\n");
+  for (int i = 0; i < service->method_count(); ++i) {
+    (*vars)["Method"] = service->method(i)->name();
+    printer->Print(*vars, "  \"/$Package$$Service$/$Method$\",\n");
+  }
+  printer->Print(*vars, "};\n\n");
+
   printer->Print(
       *vars,
       "$Service$::Stub* $Service$::NewStub("
@@ -375,15 +590,25 @@ void PrintSourceService(google::protobuf::io::Printer *printer,
       "  return stub;\n"
       "};\n\n");
   for (int i = 0; i < service->method_count(); ++i) {
+    (*vars)["Idx"] = as_string(i);
     PrintSourceClientMethod(printer, service->method(i), vars);
   }
 
+  (*vars)["MethodCount"] = as_string(service->method_count());
+  printer->Print(
+      *vars,
+      "$Service$::AsyncService::AsyncService(::grpc::CompletionQueue* cq) : "
+      "::grpc::AsynchronousService(cq, $Service$_method_names, $MethodCount$) "
+      "{}\n\n");
+
   printer->Print(*vars,
                  "$Service$::Service::~Service() {\n"
                  "  delete service_;\n"
                  "}\n\n");
   for (int i = 0; i < service->method_count(); ++i) {
+    (*vars)["Idx"] = as_string(i);
     PrintSourceServerMethod(printer, service->method(i), vars);
+    PrintSourceServerAsyncMethod(printer, service->method(i), vars);
   }
   printer->Print(*vars,
                  "::grpc::RpcService* $Service$::Service::service() {\n");
@@ -395,6 +620,7 @@ void PrintSourceService(google::protobuf::io::Printer *printer,
   printer->Print("service_ = new ::grpc::RpcService();\n");
   for (int i = 0; i < service->method_count(); ++i) {
     const google::protobuf::MethodDescriptor *method = service->method(i);
+    (*vars)["Idx"] = as_string(i);
     (*vars)["Method"] = method->name();
     (*vars)["Request"] =
         grpc_cpp_generator::ClassName(method->input_type(), true);
@@ -404,7 +630,7 @@ void PrintSourceService(google::protobuf::io::Printer *printer,
       printer->Print(
           *vars,
           "service_->AddMethod(new ::grpc::RpcServiceMethod(\n"
-          "    \"/$Package$$Service$/$Method$\",\n"
+          "    $Service$_method_names[$Idx$],\n"
           "    ::grpc::RpcMethod::NORMAL_RPC,\n"
           "    new ::grpc::RpcMethodHandler< $Service$::Service, $Request$, "
           "$Response$>(\n"
@@ -416,7 +642,7 @@ void PrintSourceService(google::protobuf::io::Printer *printer,
       printer->Print(
           *vars,
           "service_->AddMethod(new ::grpc::RpcServiceMethod(\n"
-          "    \"/$Package$$Service$/$Method$\",\n"
+          "    $Service$_method_names[$Idx$],\n"
           "    ::grpc::RpcMethod::CLIENT_STREAMING,\n"
           "    new ::grpc::ClientStreamingHandler< "
           "$Service$::Service, $Request$, $Response$>(\n"
@@ -429,7 +655,7 @@ void PrintSourceService(google::protobuf::io::Printer *printer,
       printer->Print(
           *vars,
           "service_->AddMethod(new ::grpc::RpcServiceMethod(\n"
-          "    \"/$Package$$Service$/$Method$\",\n"
+          "    $Service$_method_names[$Idx$],\n"
           "    ::grpc::RpcMethod::SERVER_STREAMING,\n"
           "    new ::grpc::ServerStreamingHandler< "
           "$Service$::Service, $Request$, $Response$>(\n"
@@ -442,7 +668,7 @@ void PrintSourceService(google::protobuf::io::Printer *printer,
       printer->Print(
           *vars,
           "service_->AddMethod(new ::grpc::RpcServiceMethod(\n"
-          "    \"/$Package$$Service$/$Method$\",\n"
+          "    $Service$_method_names[$Idx$],\n"
           "    ::grpc::RpcMethod::BIDI_STREAMING,\n"
           "    new ::grpc::BidiStreamingHandler< "
           "$Service$::Service, $Request$, $Response$>(\n"

src/core/iomgr/tcp_server.h

@@ -46,8 +46,9 @@ typedef void (*grpc_tcp_server_cb)(void *arg, grpc_endpoint *ep);
 grpc_tcp_server *grpc_tcp_server_create(void);
 
 /* Start listening to bound ports */
-void grpc_tcp_server_start(grpc_tcp_server *server, grpc_pollset *pollset,
+void grpc_tcp_server_start(grpc_tcp_server *server, grpc_pollset **pollsets,
-                           grpc_tcp_server_cb cb, void *cb_arg);
+                           size_t pollset_count, grpc_tcp_server_cb cb,
+                           void *cb_arg);
 
 /* Add a port to the server, returning port number on success, or negative
    on failure.

src/core/iomgr/tcp_server_posix.c

@@ -379,9 +379,10 @@ int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index) {
   return (index < s->nports) ? s->ports[index].fd : -1;
 }
 
-void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset *pollset,
+void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset **pollsets,
-                           grpc_tcp_server_cb cb, void *cb_arg) {
+                           size_t pollset_count, grpc_tcp_server_cb cb,
-  size_t i;
+                           void *cb_arg) {
+  size_t i, j;
   GPR_ASSERT(cb);
   gpr_mu_lock(&s->mu);
   GPR_ASSERT(!s->cb);
@@ -389,8 +390,8 @@ void grpc_tcp_server_start(grpc_tcp_server *s, grpc_pollset *pollset,
   s->cb = cb;
   s->cb_arg = cb_arg;
   for (i = 0; i < s->nports; i++) {
-    if (pollset) {
+    for (j = 0; j < pollset_count; j++) {
-      grpc_pollset_add_fd(pollset, s->ports[i].emfd);
+      grpc_pollset_add_fd(pollsets[j], s->ports[i].emfd);
     }
     grpc_fd_notify_on_read(s->ports[i].emfd, on_read, &s->ports[i]);
     s->active_ports++;

src/core/security/server_secure_chttp2.c

@@ -76,9 +76,10 @@ static void on_accept(void *server, grpc_endpoint *tcp) {
 /* Note: the following code is the same with server_chttp2.c */
 
 /* Server callback: start listening on our ports */
-static void start(grpc_server *server, void *tcpp, grpc_pollset *pollset) {
+static void start(grpc_server *server, void *tcpp, grpc_pollset **pollsets,
+                  size_t pollset_count) {
   grpc_tcp_server *tcp = tcpp;
-  grpc_tcp_server_start(tcp, pollset, on_accept, server);
+  grpc_tcp_server_start(tcp, pollsets, pollset_count, on_accept, server);
 }
 
 /* Server callback: destroy the tcp listener (so we don't generate further

src/core/statistics/census_log.c

@@ -91,9 +91,9 @@
 */
 #include "src/core/statistics/census_log.h"
 #include <string.h>
-#include "src/core/support/cpu.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
+#include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>

src/core/support/cpu_linux.c

@@ -39,7 +39,7 @@
 
 #ifdef GPR_CPU_LINUX
 
-#include "src/core/support/cpu.h"
+#include <grpc/support/cpu.h>
 
 #include <sched.h>
 #include <errno.h>

src/core/surface/call.c

@@ -258,6 +258,10 @@ void grpc_call_set_completion_queue(grpc_call *call,
   call->cq = cq;
 }
 
+grpc_completion_queue *grpc_call_get_completion_queue(grpc_call *call) {
+  return call->cq;
+}
+
 void grpc_call_internal_ref(grpc_call *c) { gpr_ref(&c->internal_refcount); }
 
 static void destroy_call(void *call, int ignored_success) {

src/core/surface/call.h

@@ -89,6 +89,7 @@ grpc_call *grpc_call_create(grpc_channel *channel, grpc_completion_queue *cq,
                             const void *server_transport_data);
 
 void grpc_call_set_completion_queue(grpc_call *call, grpc_completion_queue *cq);
+grpc_completion_queue *grpc_call_get_completion_queue(grpc_call *call);
 
 void grpc_call_internal_ref(grpc_call *call);
 void grpc_call_internal_unref(grpc_call *call, int allow_immediate_deletion);

src/core/surface/channel.c

@@ -36,6 +36,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "src/core/iomgr/iomgr.h"
 #include "src/core/surface/call.h"
 #include "src/core/surface/client.h"
 #include <grpc/support/alloc.h>
@@ -138,8 +139,8 @@ void grpc_channel_internal_ref(grpc_channel *channel) {
   gpr_ref(&channel->refs);
 }
 
-void grpc_channel_internal_unref(grpc_channel *channel) {
+static void destroy_channel(void *p, int ok) {
-  if (gpr_unref(&channel->refs)) {
+  grpc_channel *channel = p;
   grpc_channel_stack_destroy(CHANNEL_STACK_FROM_CHANNEL(channel));
   grpc_mdstr_unref(channel->grpc_status_string);
   grpc_mdstr_unref(channel->grpc_message_string);
@@ -147,6 +148,11 @@ void grpc_channel_internal_unref(grpc_channel *channel) {
   grpc_mdstr_unref(channel->authority_string);
   grpc_mdctx_orphan(channel->metadata_context);
   gpr_free(channel);
+}
+
+void grpc_channel_internal_unref(grpc_channel *channel) {
+  if (gpr_unref(&channel->refs)) {
+    grpc_iomgr_add_callback(destroy_channel, channel);
   }
 }
 

src/core/surface/lame_client.c

@@ -47,6 +47,7 @@ typedef struct {
 } call_data;
 
 typedef struct {
+  grpc_mdelem *status;
   grpc_mdelem *message;
 } channel_data;
 
@@ -57,6 +58,7 @@ static void call_op(grpc_call_element *elem, grpc_call_element *from_elem,
 
   switch (op->type) {
     case GRPC_SEND_START:
+      grpc_call_recv_metadata(elem, grpc_mdelem_ref(channeld->status));
       grpc_call_recv_metadata(elem, grpc_mdelem_ref(channeld->message));
       grpc_call_stream_closed(elem);
       break;
@@ -93,18 +95,22 @@ static void init_channel_elem(grpc_channel_element *elem,
                               const grpc_channel_args *args, grpc_mdctx *mdctx,
                               int is_first, int is_last) {
   channel_data *channeld = elem->channel_data;
+  char status[12];
 
   GPR_ASSERT(is_first);
   GPR_ASSERT(is_last);
 
   channeld->message = grpc_mdelem_from_strings(mdctx, "grpc-message",
                                                "Rpc sent on a lame channel.");
+  gpr_ltoa(GRPC_STATUS_UNKNOWN, status);
+  channeld->status = grpc_mdelem_from_strings(mdctx, "grpc-status", status);
 }
 
 static void destroy_channel_elem(grpc_channel_element *elem) {
   channel_data *channeld = elem->channel_data;
 
   grpc_mdelem_unref(channeld->message);
+  grpc_mdelem_unref(channeld->status);
 }
 
 static const grpc_channel_filter lame_filter = {

src/core/surface/server.c

@@ -53,13 +53,64 @@ typedef enum { PENDING_START, ALL_CALLS, CALL_LIST_COUNT } call_list;
 
 typedef struct listener {
   void *arg;
-  void (*start)(grpc_server *server, void *arg, grpc_pollset *pollset);
+  void (*start)(grpc_server *server, void *arg, grpc_pollset **pollsets,
+                size_t pollset_count);
   void (*destroy)(grpc_server *server, void *arg);
   struct listener *next;
 } listener;
 
 typedef struct call_data call_data;
 typedef struct channel_data channel_data;
+typedef struct registered_method registered_method;
+
+typedef struct {
+  call_data *next;
+  call_data *prev;
+} call_link;
+
+typedef enum { LEGACY_CALL, BATCH_CALL, REGISTERED_CALL } requested_call_type;
+
+typedef struct {
+  requested_call_type type;
+  void *tag;
+  union {
+    struct {
+      grpc_completion_queue *cq_bind;
+      grpc_call **call;
+      grpc_call_details *details;
+      grpc_metadata_array *initial_metadata;
+    } batch;
+    struct {
+      grpc_completion_queue *cq_bind;
+      grpc_call **call;
+      registered_method *registered_method;
+      gpr_timespec *deadline;
+      grpc_metadata_array *initial_metadata;
+      grpc_byte_buffer **optional_payload;
+    } registered;
+  } data;
+} requested_call;
+
+typedef struct {
+  requested_call *calls;
+  size_t count;
+  size_t capacity;
+} requested_call_array;
+
+struct registered_method {
+  char *method;
+  char *host;
+  call_data *pending;
+  requested_call_array requested;
+  grpc_completion_queue *cq;
+  registered_method *next;
+};
+
+typedef struct channel_registered_method {
+  registered_method *server_registered_method;
+  grpc_mdstr *method;
+  grpc_mdstr *host;
+} channel_registered_method;
 
 struct channel_data {
   grpc_server *server;
@@ -69,37 +120,29 @@ struct channel_data {
   /* linked list of all channels on a server */
   channel_data *next;
   channel_data *prev;
+  channel_registered_method *registered_methods;
+  gpr_uint32 registered_method_slots;
+  gpr_uint32 registered_method_max_probes;
 };
 
-typedef void (*new_call_cb)(grpc_server *server, grpc_completion_queue *cq,
-                            grpc_call **call, grpc_call_details *details,
-                            grpc_metadata_array *initial_metadata,
-                            call_data *calld, void *user_data);
-
-typedef struct {
-  void *user_data;
-  grpc_completion_queue *cq;
-  grpc_call **call;
-  grpc_call_details *details;
-  grpc_metadata_array *initial_metadata;
-  new_call_cb cb;
-} requested_call;
-
 struct grpc_server {
   size_t channel_filter_count;
   const grpc_channel_filter **channel_filters;
   grpc_channel_args *channel_args;
-  grpc_completion_queue *cq;
+  grpc_completion_queue *unregistered_cq;
+
+  grpc_completion_queue **cqs;
+  grpc_pollset **pollsets;
+  size_t cq_count;
 
   gpr_mu mu;
 
-  requested_call *requested_calls;
+  registered_method *registered_methods;
-  size_t requested_call_count;
+  requested_call_array requested_calls;
-  size_t requested_call_capacity;
 
   gpr_uint8 shutdown;
-  gpr_uint8 have_shutdown_tag;
+  size_t num_shutdown_tags;
-  void *shutdown_tag;
+  void **shutdown_tags;
 
   call_data *lists[CALL_LIST_COUNT];
   channel_data root_channel_data;
@@ -108,11 +151,6 @@ struct grpc_server {
   gpr_refcount internal_refcount;
 };
 
-typedef struct {
-  call_data *next;
-  call_data *prev;
-} call_link;
-
 typedef enum {
   /* waiting for metadata */
   NOT_STARTED,
@@ -125,7 +163,7 @@ typedef enum {
 } call_state;
 
 typedef struct legacy_data {
-  grpc_metadata_array *initial_metadata;
+  grpc_metadata_array initial_metadata;
 } legacy_data;
 
 struct call_data {
@@ -137,9 +175,9 @@ struct call_data {
   grpc_mdstr *host;
 
   legacy_data *legacy;
-  grpc_call_details *details;
+  grpc_completion_queue *cq_new;
 
-  gpr_uint8 included[CALL_LIST_COUNT];
+  call_data **root[CALL_LIST_COUNT];
   call_link links[CALL_LIST_COUNT];
 };
 
@@ -148,30 +186,33 @@ struct call_data {
 
 static void do_nothing(void *unused, grpc_op_error ignored) {}
 
-static int call_list_join(grpc_server *server, call_data *call,
+static void begin_call(grpc_server *server, call_data *calld,
-                          call_list list) {
+                       requested_call *rc);
-  if (call->included[list]) return 0;
+static void fail_call(grpc_server *server, requested_call *rc);
-  call->included[list] = 1;
+
-  if (!server->lists[list]) {
+static int call_list_join(call_data **root, call_data *call, call_list list) {
-    server->lists[list] = call;
+  GPR_ASSERT(!call->root[list]);
+  call->root[list] = root;
+  if (!*root) {
+    *root = call;
     call->links[list].next = call->links[list].prev = call;
   } else {
-    call->links[list].next = server->lists[list];
+    call->links[list].next = *root;
-    call->links[list].prev = server->lists[list]->links[list].prev;
+    call->links[list].prev = (*root)->links[list].prev;
     call->links[list].next->links[list].prev =
         call->links[list].prev->links[list].next = call;
   }
   return 1;
 }
 
-static call_data *call_list_remove_head(grpc_server *server, call_list list) {
+static call_data *call_list_remove_head(call_data **root, call_list list) {
-  call_data *out = server->lists[list];
+  call_data *out = *root;
   if (out) {
-    out->included[list] = 0;
+    out->root[list] = NULL;
     if (out->links[list].next == out) {
-      server->lists[list] = NULL;
+      *root = NULL;
     } else {
-      server->lists[list] = out->links[list].next;
+      *root = out->links[list].next;
       out->links[list].next->links[list].prev = out->links[list].prev;
       out->links[list].prev->links[list].next = out->links[list].next;
     }
@@ -179,33 +220,60 @@ static call_data *call_list_remove_head(grpc_server *server, call_list list) {
   return out;
 }
 
-static int call_list_remove(grpc_server *server, call_data *call,
+static int call_list_remove(call_data *call, call_list list) {
-                            call_list list) {
+  call_data **root = call->root[list];
-  if (!call->included[list]) return 0;
+  if (root == NULL) return 0;
-  call->included[list] = 0;
+  call->root[list] = NULL;
-  if (server->lists[list] == call) {
+  if (*root == call) {
-    server->lists[list] = call->links[list].next;
+    *root = call->links[list].next;
-    if (server->lists[list] == call) {
+    if (*root == call) {
-      server->lists[list] = NULL;
+      *root = NULL;
       return 1;
     }
   }
-  GPR_ASSERT(server->lists[list] != call);
+  GPR_ASSERT(*root != call);
   call->links[list].next->links[list].prev = call->links[list].prev;
   call->links[list].prev->links[list].next = call->links[list].next;
   return 1;
 }
 
+static void requested_call_array_destroy(requested_call_array *array) {
+  gpr_free(array->calls);
+}
+
+static requested_call *requested_call_array_add(requested_call_array *array) {
+  requested_call *rc;
+  if (array->count == array->capacity) {
+    array->capacity = GPR_MAX(array->capacity + 8, array->capacity * 2);
+    array->calls =
+        gpr_realloc(array->calls, sizeof(requested_call) * array->capacity);
+  }
+  rc = &array->calls[array->count++];
+  memset(rc, 0, sizeof(*rc));
+  return rc;
+}
+
 static void server_ref(grpc_server *server) {
   gpr_ref(&server->internal_refcount);
 }
 
 static void server_unref(grpc_server *server) {
+  registered_method *rm;
   if (gpr_unref(&server->internal_refcount)) {
     grpc_channel_args_destroy(server->channel_args);
     gpr_mu_destroy(&server->mu);
     gpr_free(server->channel_filters);
-    gpr_free(server->requested_calls);
+    requested_call_array_destroy(&server->requested_calls);
+    while ((rm = server->registered_methods) != NULL) {
+      server->registered_methods = rm->next;
+      gpr_free(rm->method);
+      gpr_free(rm->host);
+      requested_call_array_destroy(&rm->requested);
+      gpr_free(rm);
+    }
+    gpr_free(server->cqs);
+    gpr_free(server->pollsets);
+    gpr_free(server->shutdown_tags);
     gpr_free(server);
   }
 }
@@ -223,7 +291,6 @@ static void orphan_channel(channel_data *chand) {
 static void finish_destroy_channel(void *cd, int success) {
   channel_data *chand = cd;
   grpc_server *server = chand->server;
-  /*gpr_log(GPR_INFO, "destroy channel %p", chand->channel);*/
   grpc_channel_destroy(chand->channel);
   server_unref(server);
 }
@@ -236,23 +303,64 @@ static void destroy_channel(channel_data *chand) {
   grpc_iomgr_add_callback(finish_destroy_channel, chand);
 }
 
+static void finish_start_new_rpc_and_unlock(grpc_server *server,
+                                            grpc_call_element *elem,
+                                            call_data **pending_root,
+                                            requested_call_array *array) {
+  requested_call rc;
+  call_data *calld = elem->call_data;
+  if (array->count == 0) {
+    calld->state = PENDING;
+    call_list_join(pending_root, calld, PENDING_START);
+    gpr_mu_unlock(&server->mu);
+  } else {
+    rc = array->calls[--array->count];
+    calld->state = ACTIVATED;
+    gpr_mu_unlock(&server->mu);
+    begin_call(server, calld, &rc);
+  }
+}
+
 static void start_new_rpc(grpc_call_element *elem) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   grpc_server *server = chand->server;
+  gpr_uint32 i;
+  gpr_uint32 hash;
+  channel_registered_method *rm;
 
   gpr_mu_lock(&server->mu);
-  if (server->requested_call_count > 0) {
+  if (chand->registered_methods && calld->path && calld->host) {
-    requested_call rc = server->requested_calls[--server->requested_call_count];
+    /* TODO(ctiller): unify these two searches */
-    calld->state = ACTIVATED;
+    /* check for an exact match with host */
-    gpr_mu_unlock(&server->mu);
+    hash = GRPC_MDSTR_KV_HASH(calld->host->hash, calld->path->hash);
-    rc.cb(server, rc.cq, rc.call, rc.details, rc.initial_metadata, calld,
+    for (i = 0; i < chand->registered_method_max_probes; i++) {
-          rc.user_data);
+      rm = &chand->registered_methods[(hash + i) %
-  } else {
+                                      chand->registered_method_slots];
-    calld->state = PENDING;
+      if (!rm) break;
-    call_list_join(server, calld, PENDING_START);
+      if (rm->host != calld->host) continue;
-    gpr_mu_unlock(&server->mu);
+      if (rm->method != calld->path) continue;
+      finish_start_new_rpc_and_unlock(server, elem,
+                                      &rm->server_registered_method->pending,
+                                      &rm->server_registered_method->requested);
+      return;
+    }
+    /* check for a wildcard method definition (no host set) */
+    hash = GRPC_MDSTR_KV_HASH(0, calld->path->hash);
+    for (i = 0; i <= chand->registered_method_max_probes; i++) {
+      rm = &chand->registered_methods[(hash + i) %
+                                      chand->registered_method_slots];
+      if (!rm) break;
+      if (rm->host != NULL) continue;
+      if (rm->method != calld->path) continue;
+      finish_start_new_rpc_and_unlock(server, elem,
+                                      &rm->server_registered_method->pending,
+                                      &rm->server_registered_method->requested);
+      return;
     }
+  }
+  finish_start_new_rpc_and_unlock(server, elem, &server->lists[PENDING_START],
+                                  &server->requested_calls);
 }
 
 static void kill_zombie(void *elem, int success) {
@@ -267,7 +375,7 @@ static void stream_closed(grpc_call_element *elem) {
     case ACTIVATED:
       break;
     case PENDING:
-      call_list_remove(chand->server, calld, PENDING_START);
+      call_list_remove(calld, PENDING_START);
     /* fallthrough intended */
     case NOT_STARTED:
       calld->state = ZOMBIED;
@@ -398,7 +506,7 @@ static void init_call_elem(grpc_call_element *elem,
   calld->call = grpc_call_from_top_element(elem);
 
   gpr_mu_lock(&chand->server->mu);
-  call_list_join(chand->server, calld, ALL_CALLS);
+  call_list_join(&chand->server->lists[ALL_CALLS], calld, ALL_CALLS);
   gpr_mu_unlock(&chand->server->mu);
 
   server_ref(chand->server);
@@ -407,15 +515,19 @@ static void init_call_elem(grpc_call_element *elem,
 static void destroy_call_elem(grpc_call_element *elem) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  int i;
+  size_t i, j;
 
   gpr_mu_lock(&chand->server->mu);
   for (i = 0; i < CALL_LIST_COUNT; i++) {
-    call_list_remove(chand->server, elem->call_data, i);
+    call_list_remove(elem->call_data, i);
+  }
+  if (chand->server->shutdown && chand->server->lists[ALL_CALLS] == NULL) {
+    for (i = 0; i < chand->server->num_shutdown_tags; i++) {
+      for (j = 0; j < chand->server->cq_count; j++) {
+        grpc_cq_end_server_shutdown(chand->server->cqs[j],
+                                    chand->server->shutdown_tags[i]);
+      }
     }
-  if (chand->server->shutdown && chand->server->have_shutdown_tag &&
-      chand->server->lists[ALL_CALLS] == NULL) {
-    grpc_cq_end_server_shutdown(chand->server->cq, chand->server->shutdown_tag);
   }
   gpr_mu_unlock(&chand->server->mu);
 
@@ -427,8 +539,7 @@ static void destroy_call_elem(grpc_call_element *elem) {
   }
 
   if (calld->legacy) {
-    gpr_free(calld->legacy->initial_metadata->metadata);
+    gpr_free(calld->legacy->initial_metadata.metadata);
-    gpr_free(calld->legacy->initial_metadata);
     gpr_free(calld->legacy);
   }
 
@@ -447,10 +558,23 @@ static void init_channel_elem(grpc_channel_element *elem,
   chand->path_key = grpc_mdstr_from_string(metadata_context, ":path");
   chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority");
   chand->next = chand->prev = chand;
+  chand->registered_methods = NULL;
 }
 
 static void destroy_channel_elem(grpc_channel_element *elem) {
+  size_t i;
   channel_data *chand = elem->channel_data;
+  if (chand->registered_methods) {
+    for (i = 0; i < chand->registered_method_slots; i++) {
+      if (chand->registered_methods[i].method) {
+        grpc_mdstr_unref(chand->registered_methods[i].method);
+      }
+      if (chand->registered_methods[i].host) {
+        grpc_mdstr_unref(chand->registered_methods[i].host);
+      }
+    }
+    gpr_free(chand->registered_methods);
+  }
   if (chand->server) {
     gpr_mu_lock(&chand->server->mu);
     chand->next->prev = chand->prev;
@@ -469,6 +593,17 @@ static const grpc_channel_filter server_surface_filter = {
     init_channel_elem, destroy_channel_elem, "server",
 };
 
+static void addcq(grpc_server *server, grpc_completion_queue *cq) {
+  size_t i, n;
+  for (i = 0; i < server->cq_count; i++) {
+    if (server->cqs[i] == cq) return;
+  }
+  n = server->cq_count++;
+  server->cqs = gpr_realloc(server->cqs,
+                            server->cq_count * sizeof(grpc_completion_queue *));
+  server->cqs[n] = cq;
+}
+
 grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
                                              grpc_channel_filter **filters,
                                              size_t filter_count,
@@ -478,10 +613,11 @@ grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
 
   grpc_server *server = gpr_malloc(sizeof(grpc_server));
   memset(server, 0, sizeof(grpc_server));
+  if (cq) addcq(server, cq);
 
   gpr_mu_init(&server->mu);
 
-  server->cq = cq;
+  server->unregistered_cq = cq;
   /* decremented by grpc_server_destroy */
   gpr_ref_init(&server->internal_refcount, 1);
   server->root_channel_data.next = server->root_channel_data.prev =
@@ -509,11 +645,50 @@ grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
   return server;
 }
 
+static int streq(const char *a, const char *b) {
+  if (a == NULL && b == NULL) return 1;
+  if (a == NULL) return 0;
+  if (b == NULL) return 0;
+  return 0 == strcmp(a, b);
+}
+
+void *grpc_server_register_method(grpc_server *server, const char *method,
+                                  const char *host,
+                                  grpc_completion_queue *cq_new_rpc) {
+  registered_method *m;
+  if (!method) {
+    gpr_log(GPR_ERROR, "%s method string cannot be NULL", __FUNCTION__);
+    return NULL;
+  }
+  for (m = server->registered_methods; m; m = m->next) {
+    if (streq(m->method, method) && streq(m->host, host)) {
+      gpr_log(GPR_ERROR, "duplicate registration for %s@%s", method,
+              host ? host : "*");
+      return NULL;
+    }
+  }
+  addcq(server, cq_new_rpc);
+  m = gpr_malloc(sizeof(registered_method));
+  memset(m, 0, sizeof(*m));
+  m->method = gpr_strdup(method);
+  m->host = gpr_strdup(host);
+  m->next = server->registered_methods;
+  m->cq = cq_new_rpc;
+  server->registered_methods = m;
+  return m;
+}
+
 void grpc_server_start(grpc_server *server) {
   listener *l;
+  size_t i;
+
+  server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
+  for (i = 0; i < server->cq_count; i++) {
+    server->pollsets[i] = grpc_cq_pollset(server->cqs[i]);
+  }
 
   for (l = server->listeners; l; l = l->next) {
-    l->start(server, l->arg, grpc_cq_pollset(server->cq));
+    l->start(server, l->arg, server->pollsets, server->cq_count);
   }
 }
 
@@ -525,8 +700,19 @@ grpc_transport_setup_result grpc_server_setup_transport(
   grpc_channel_filter const **filters =
       gpr_malloc(sizeof(grpc_channel_filter *) * num_filters);
   size_t i;
+  size_t num_registered_methods;
+  size_t alloc;
+  registered_method *rm;
+  channel_registered_method *crm;
   grpc_channel *channel;
   channel_data *chand;
+  grpc_mdstr *host;
+  grpc_mdstr *method;
+  gpr_uint32 hash;
+  gpr_uint32 slots;
+  gpr_uint32 probes;
+  gpr_uint32 max_probes = 0;
+  grpc_transport_setup_result result;
 
   for (i = 0; i < s->channel_filter_count; i++) {
     filters[i] = s->channel_filters[i];
@@ -536,7 +722,9 @@ grpc_transport_setup_result grpc_server_setup_transport(
   }
   filters[i] = &grpc_connected_channel_filter;
 
-  grpc_transport_add_to_pollset(transport, grpc_cq_pollset(s->cq));
+  for (i = 0; i < s->cq_count; i++) {
+    grpc_transport_add_to_pollset(transport, grpc_cq_pollset(s->cqs[i]));
+  }
 
   channel = grpc_channel_create_from_filters(filters, num_filters,
                                              s->channel_args, mdctx, 0);
@@ -546,6 +734,38 @@ grpc_transport_setup_result grpc_server_setup_transport(
   server_ref(s);
   chand->channel = channel;
 
+  num_registered_methods = 0;
+  for (rm = s->registered_methods; rm; rm = rm->next) {
+    num_registered_methods++;
+  }
+  /* build a lookup table phrased in terms of mdstr's in this channels context
+     to quickly find registered methods */
+  if (num_registered_methods > 0) {
+    slots = 2 * num_registered_methods;
+    alloc = sizeof(channel_registered_method) * slots;
+    chand->registered_methods = gpr_malloc(alloc);
+    memset(chand->registered_methods, 0, alloc);
+    for (rm = s->registered_methods; rm; rm = rm->next) {
+      host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host) : NULL;
+      method = grpc_mdstr_from_string(mdctx, rm->method);
+      hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
+      for (probes = 0; chand->registered_methods[(hash + probes) % slots]
+                               .server_registered_method != NULL;
+           probes++)
+        ;
+      if (probes > max_probes) max_probes = probes;
+      crm = &chand->registered_methods[(hash + probes) % slots];
+      crm->server_registered_method = rm;
+      crm->host = host;
+      crm->method = method;
+    }
+    chand->registered_method_slots = slots;
+    chand->registered_method_max_probes = max_probes;
+  }
+
+  result = grpc_connected_channel_bind_transport(
+      grpc_channel_get_channel_stack(channel), transport);
+
   gpr_mu_lock(&s->mu);
   chand->next = &s->root_channel_data;
   chand->prev = chand->next->prev;
@@ -554,24 +774,32 @@ grpc_transport_setup_result grpc_server_setup_transport(
 
   gpr_free(filters);
 
-  return grpc_connected_channel_bind_transport(
+  return result;
-      grpc_channel_get_channel_stack(channel), transport);
 }
 
-void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
+static void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
                               void *shutdown_tag) {
   listener *l;
-  requested_call *requested_calls;
+  requested_call_array requested_calls;
-  size_t requested_call_count;
   channel_data **channels;
   channel_data *c;
   size_t nchannels;
-  size_t i;
+  size_t i, j;
   grpc_channel_op op;
   grpc_channel_element *elem;
+  registered_method *rm;
 
   /* lock, and gather up some stuff to do */
   gpr_mu_lock(&server->mu);
+  if (have_shutdown_tag) {
+    for (i = 0; i < server->cq_count; i++) {
+      grpc_cq_begin_op(server->cqs[i], NULL, GRPC_SERVER_SHUTDOWN);
+    }
+    server->shutdown_tags =
+        gpr_realloc(server->shutdown_tags,
+                    sizeof(void *) * (server->num_shutdown_tags + 1));
+    server->shutdown_tags[server->num_shutdown_tags++] = shutdown_tag;
+  }
   if (server->shutdown) {
     gpr_mu_unlock(&server->mu);
     return;
@@ -591,18 +819,32 @@ void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
     i++;
   }
 
+  /* collect all unregistered then registered calls */
   requested_calls = server->requested_calls;
-  requested_call_count = server->requested_call_count;
+  memset(&server->requested_calls, 0, sizeof(server->requested_calls));
-  server->requested_calls = NULL;
+  for (rm = server->registered_methods; rm; rm = rm->next) {
-  server->requested_call_count = 0;
+    if (requested_calls.count + rm->requested.count >
+        requested_calls.capacity) {
+      requested_calls.capacity =
+          GPR_MAX(requested_calls.count + rm->requested.count,
+                  2 * requested_calls.capacity);
+      requested_calls.calls =
+          gpr_realloc(requested_calls.calls, sizeof(*requested_calls.calls) *
+                                                 requested_calls.capacity);
+    }
+    memcpy(requested_calls.calls + requested_calls.count, rm->requested.calls,
+           sizeof(*requested_calls.calls) * rm->requested.count);
+    requested_calls.count += rm->requested.count;
+    gpr_free(rm->requested.calls);
+    memset(&rm->requested, 0, sizeof(rm->requested));
+  }
 
   server->shutdown = 1;
-  server->have_shutdown_tag = have_shutdown_tag;
-  server->shutdown_tag = shutdown_tag;
-  if (have_shutdown_tag) {
-    grpc_cq_begin_op(server->cq, NULL, GRPC_SERVER_SHUTDOWN);
   if (server->lists[ALL_CALLS] == NULL) {
-      grpc_cq_end_server_shutdown(server->cq, shutdown_tag);
+    for (i = 0; i < server->num_shutdown_tags; i++) {
+      for (j = 0; j < server->cq_count; j++) {
+        grpc_cq_end_server_shutdown(server->cqs[j], server->shutdown_tags[i]);
+      }
     }
   }
   gpr_mu_unlock(&server->mu);
@@ -623,13 +865,10 @@ void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
   gpr_free(channels);
 
   /* terminate all the requested calls */
-  for (i = 0; i < requested_call_count; i++) {
+  for (i = 0; i < requested_calls.count; i++) {
-    requested_calls[i].cb(server, requested_calls[i].cq,
+    fail_call(server, &requested_calls.calls[i]);
-                          requested_calls[i].call, requested_calls[i].details,
-                          requested_calls[i].initial_metadata, NULL,
-                          requested_calls[i].user_data);
   }
-  gpr_free(requested_calls);
+  gpr_free(requested_calls.calls);
 
   /* Shutdown listeners */
   for (l = server->listeners; l; l = l->next) {
@@ -653,6 +892,12 @@ void grpc_server_shutdown_and_notify(grpc_server *server, void *tag) {
 void grpc_server_destroy(grpc_server *server) {
   channel_data *c;
   gpr_mu_lock(&server->mu);
+  if (!server->shutdown) {
+    gpr_mu_unlock(&server->mu);
+    grpc_server_shutdown(server);
+    gpr_mu_lock(&server->mu);
+  }
+
   for (c = server->root_channel_data.next; c != &server->root_channel_data;
        c = c->next) {
     shutdown_channel(c);
@@ -664,7 +909,8 @@ void grpc_server_destroy(grpc_server *server) {
 
 void grpc_server_add_listener(grpc_server *server, void *arg,
                               void (*start)(grpc_server *server, void *arg,
-                                            grpc_pollset *pollset),
+                                            grpc_pollset **pollsets,
+                                            size_t pollset_count),
                               void (*destroy)(grpc_server *server, void *arg)) {
   listener *l = gpr_malloc(sizeof(listener));
   l->arg = arg;
@@ -675,47 +921,92 @@ void grpc_server_add_listener(grpc_server *server, void *arg,
 }
 
 static grpc_call_error queue_call_request(grpc_server *server,
-                                          grpc_completion_queue *cq,
+                                          requested_call *rc) {
-                                          grpc_call **call,
+  call_data *calld = NULL;
-                                          grpc_call_details *details,
+  requested_call_array *requested_calls = NULL;
-                                          grpc_metadata_array *initial_metadata,
-                                          new_call_cb cb, void *user_data) {
-  call_data *calld;
-  requested_call *rc;
   gpr_mu_lock(&server->mu);
   if (server->shutdown) {
     gpr_mu_unlock(&server->mu);
-    cb(server, cq, call, details, initial_metadata, NULL, user_data);
+    fail_call(server, rc);
     return GRPC_CALL_OK;
   }
-  calld = call_list_remove_head(server, PENDING_START);
+  switch (rc->type) {
+    case LEGACY_CALL:
+    case BATCH_CALL:
+      calld =
+          call_list_remove_head(&server->lists[PENDING_START], PENDING_START);
+      requested_calls = &server->requested_calls;
+      break;
+    case REGISTERED_CALL:
+      calld = call_list_remove_head(
+          &rc->data.registered.registered_method->pending, PENDING_START);
+      requested_calls = &rc->data.registered.registered_method->requested;
+      break;
+  }
   if (calld) {
     GPR_ASSERT(calld->state == PENDING);
     calld->state = ACTIVATED;
     gpr_mu_unlock(&server->mu);
-    cb(server, cq, call, details, initial_metadata, calld, user_data);
+    begin_call(server, calld, rc);
     return GRPC_CALL_OK;
   } else {
-    if (server->requested_call_count == server->requested_call_capacity) {
+    *requested_call_array_add(requested_calls) = *rc;
-      server->requested_call_capacity =
-          GPR_MAX(server->requested_call_capacity + 8,
-                  server->requested_call_capacity * 2);
-      server->requested_calls =
-          gpr_realloc(server->requested_calls,
-                      sizeof(requested_call) * server->requested_call_capacity);
-    }
-    rc = &server->requested_calls[server->requested_call_count++];
-    rc->cb = cb;
-    rc->cq = cq;
-    rc->call = call;
-    rc->details = details;
-    rc->user_data = user_data;
-    rc->initial_metadata = initial_metadata;
     gpr_mu_unlock(&server->mu);
     return GRPC_CALL_OK;
   }
 }
 
+grpc_call_error grpc_server_request_call(grpc_server *server, grpc_call **call,
+                                         grpc_call_details *details,
+                                         grpc_metadata_array *initial_metadata,
+                                         grpc_completion_queue *cq_bind,
+                                         void *tag) {
+  requested_call rc;
+  grpc_cq_begin_op(server->unregistered_cq, NULL, GRPC_OP_COMPLETE);
+  rc.type = BATCH_CALL;
+  rc.tag = tag;
+  rc.data.batch.cq_bind = cq_bind;
+  rc.data.batch.call = call;
+  rc.data.batch.details = details;
+  rc.data.batch.initial_metadata = initial_metadata;
+  return queue_call_request(server, &rc);
+}
+
+grpc_call_error grpc_server_request_registered_call(
+    grpc_server *server, void *rm, grpc_call **call, gpr_timespec *deadline,
+    grpc_metadata_array *initial_metadata, grpc_byte_buffer **optional_payload,
+    grpc_completion_queue *cq_bind, void *tag) {
+  requested_call rc;
+  registered_method *registered_method = rm;
+  grpc_cq_begin_op(registered_method->cq, NULL, GRPC_OP_COMPLETE);
+  rc.type = REGISTERED_CALL;
+  rc.tag = tag;
+  rc.data.registered.cq_bind = cq_bind;
+  rc.data.registered.call = call;
+  rc.data.registered.registered_method = registered_method;
+  rc.data.registered.deadline = deadline;
+  rc.data.registered.initial_metadata = initial_metadata;
+  rc.data.registered.optional_payload = optional_payload;
+  return queue_call_request(server, &rc);
+}
+
+grpc_call_error grpc_server_request_call_old(grpc_server *server,
+                                             void *tag_new) {
+  requested_call rc;
+  grpc_cq_begin_op(server->unregistered_cq, NULL, GRPC_SERVER_RPC_NEW);
+  rc.type = LEGACY_CALL;
+  rc.tag = tag_new;
+  return queue_call_request(server, &rc);
+}
+
+static void publish_legacy(grpc_call *call, grpc_op_error status, void *tag);
+static void publish_registered_or_batch(grpc_call *call, grpc_op_error status,
+                                        void *tag);
+static void publish_was_not_set(grpc_call *call, grpc_op_error status,
+                                void *tag) {
+  abort();
+}
+
 static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) {
   gpr_slice slice = value->slice;
   size_t len = GPR_SLICE_LENGTH(slice);
@@ -727,57 +1018,84 @@ static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) {
   memcpy(*dest, grpc_mdstr_as_c_string(value), len + 1);
 }
 
-static void publish_request(grpc_call *call, grpc_op_error status, void *tag) {
+static void begin_call(grpc_server *server, call_data *calld,
-  grpc_call_element *elem =
+                       requested_call *rc) {
-      grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
+  grpc_ioreq_completion_func publish = publish_was_not_set;
-  call_data *calld = elem->call_data;
+  grpc_ioreq req[2];
-  channel_data *chand = elem->channel_data;
+  grpc_ioreq *r = req;
-  grpc_server *server = chand->server;
 
-  if (status == GRPC_OP_OK) {
+  /* called once initial metadata has been read by the call, but BEFORE
-    cpstr(&calld->details->host, &calld->details->host_capacity, calld->host);
+     the ioreq to fetch it out of the call has been executed.
-    cpstr(&calld->details->method, &calld->details->method_capacity,
+     This means metadata related fields can be relied on in calld, but to
-          calld->path);
+     fill in the metadata array passed by the client, we need to perform
-    calld->details->deadline = calld->deadline;
+     an ioreq op, that should complete immediately. */
-    grpc_cq_end_op_complete(server->cq, tag, call, do_nothing, NULL,
-                            GRPC_OP_OK);
-  } else {
-    abort();
-  }
-}
 
-static void begin_request(grpc_server *server, grpc_completion_queue *cq,
+  switch (rc->type) {
-                          grpc_call **call, grpc_call_details *details,
+    case LEGACY_CALL:
-                          grpc_metadata_array *initial_metadata,
+      calld->legacy = gpr_malloc(sizeof(legacy_data));
-                          call_data *calld, void *tag) {
+      memset(calld->legacy, 0, sizeof(legacy_data));
-  grpc_ioreq req;
+      r->op = GRPC_IOREQ_RECV_INITIAL_METADATA;
-  if (!calld) {
+      r->data.recv_metadata = &calld->legacy->initial_metadata;
-    *call = NULL;
+      r++;
-    initial_metadata->count = 0;
+      publish = publish_legacy;
-    grpc_cq_end_op_complete(cq, tag, NULL, do_nothing, NULL, GRPC_OP_ERROR);
+      break;
-    return;
+    case BATCH_CALL:
+      cpstr(&rc->data.batch.details->host,
+            &rc->data.batch.details->host_capacity, calld->host);
+      cpstr(&rc->data.batch.details->method,
+            &rc->data.batch.details->method_capacity, calld->path);
+      grpc_call_set_completion_queue(calld->call, rc->data.batch.cq_bind);
+      *rc->data.batch.call = calld->call;
+      r->op = GRPC_IOREQ_RECV_INITIAL_METADATA;
+      r->data.recv_metadata = rc->data.batch.initial_metadata;
+      r++;
+      calld->cq_new = server->unregistered_cq;
+      publish = publish_registered_or_batch;
+      break;
+    case REGISTERED_CALL:
+      *rc->data.registered.deadline = calld->deadline;
+      grpc_call_set_completion_queue(calld->call, rc->data.registered.cq_bind);
+      *rc->data.registered.call = calld->call;
+      r->op = GRPC_IOREQ_RECV_INITIAL_METADATA;
+      r->data.recv_metadata = rc->data.registered.initial_metadata;
+      r++;
+      if (rc->data.registered.optional_payload) {
+        r->op = GRPC_IOREQ_RECV_MESSAGE;
+        r->data.recv_message = rc->data.registered.optional_payload;
+        r++;
+      }
+      calld->cq_new = rc->data.registered.registered_method->cq;
+      publish = publish_registered_or_batch;
+      break;
   }
-  calld->details = details;
+
-  grpc_call_set_completion_queue(calld->call, cq);
-  *call = calld->call;
-  req.op = GRPC_IOREQ_RECV_INITIAL_METADATA;
-  req.data.recv_metadata = initial_metadata;
   grpc_call_internal_ref(calld->call);
-  grpc_call_start_ioreq_and_call_back(calld->call, &req, 1, publish_request,
+  grpc_call_start_ioreq_and_call_back(calld->call, req, r - req, publish,
-                                      tag);
+                                      rc->tag);
 }
 
-grpc_call_error grpc_server_request_call(grpc_server *server, grpc_call **call,
+static void fail_call(grpc_server *server, requested_call *rc) {
-                                         grpc_call_details *details,
+  switch (rc->type) {
-                                         grpc_metadata_array *initial_metadata,
+    case LEGACY_CALL:
-                                         grpc_completion_queue *cq, void *tag) {
+      grpc_cq_end_new_rpc(server->unregistered_cq, rc->tag, NULL, do_nothing,
-  grpc_cq_begin_op(cq, NULL, GRPC_OP_COMPLETE);
+                          NULL, NULL, NULL, gpr_inf_past, 0, NULL);
-  return queue_call_request(server, cq, call, details, initial_metadata,
+      break;
-                            begin_request, tag);
+    case BATCH_CALL:
+      *rc->data.batch.call = NULL;
+      rc->data.batch.initial_metadata->count = 0;
+      grpc_cq_end_op_complete(server->unregistered_cq, rc->tag, NULL,
+                              do_nothing, NULL, GRPC_OP_ERROR);
+      break;
+    case REGISTERED_CALL:
+      *rc->data.registered.call = NULL;
+      rc->data.registered.initial_metadata->count = 0;
+      grpc_cq_end_op_complete(rc->data.registered.registered_method->cq,
+                              rc->tag, NULL, do_nothing, NULL, GRPC_OP_ERROR);
+      break;
+  }
 }
 
-static void publish_legacy_request(grpc_call *call, grpc_op_error status,
+static void publish_legacy(grpc_call *call, grpc_op_error status, void *tag) {
-                                   void *tag) {
   grpc_call_element *elem =
       grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
   call_data *calld = elem->call_data;
@@ -785,47 +1103,23 @@ static void publish_legacy_request(grpc_call *call, grpc_op_error status,
   grpc_server *server = chand->server;
 
   if (status == GRPC_OP_OK) {
-    grpc_cq_end_new_rpc(server->cq, tag, call, do_nothing, NULL,
+    grpc_cq_end_new_rpc(server->unregistered_cq, tag, call, do_nothing, NULL,
                         grpc_mdstr_as_c_string(calld->path),
                         grpc_mdstr_as_c_string(calld->host), calld->deadline,
-                        calld->legacy->initial_metadata->count,
+                        calld->legacy->initial_metadata.count,
-                        calld->legacy->initial_metadata->metadata);
+                        calld->legacy->initial_metadata.metadata);
   } else {
+    gpr_log(GPR_ERROR, "should never reach here");
     abort();
   }
 }
 
-static void begin_legacy_request(grpc_server *server, grpc_completion_queue *cq,
+static void publish_registered_or_batch(grpc_call *call, grpc_op_error status,
-                                 grpc_call **call, grpc_call_details *details,
+                                        void *tag) {
-                                 grpc_metadata_array *initial_metadata,
+  grpc_call_element *elem =
-                                 call_data *calld, void *tag) {
+      grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
-  grpc_ioreq req;
+  call_data *calld = elem->call_data;
-  GPR_ASSERT(call == NULL);
+  grpc_cq_end_op_complete(calld->cq_new, tag, call, do_nothing, NULL, status);
-  GPR_ASSERT(details == NULL);
-  if (!calld) {
-    gpr_free(initial_metadata);
-    grpc_cq_end_new_rpc(cq, tag, NULL, do_nothing, NULL, NULL, NULL,
-                        gpr_inf_past, 0, NULL);
-    return;
-  }
-  req.op = GRPC_IOREQ_RECV_INITIAL_METADATA;
-  req.data.recv_metadata = initial_metadata;
-  calld->legacy = gpr_malloc(sizeof(legacy_data));
-  memset(calld->legacy, 0, sizeof(legacy_data));
-  calld->legacy->initial_metadata = initial_metadata;
-  grpc_call_internal_ref(calld->call);
-  grpc_call_start_ioreq_and_call_back(calld->call, &req, 1,
-                                      publish_legacy_request, tag);
-}
-
-grpc_call_error grpc_server_request_call_old(grpc_server *server,
-                                             void *tag_new) {
-  grpc_metadata_array *client_metadata =
-      gpr_malloc(sizeof(grpc_metadata_array));
-  memset(client_metadata, 0, sizeof(*client_metadata));
-  grpc_cq_begin_op(server->cq, NULL, GRPC_SERVER_RPC_NEW);
-  return queue_call_request(server, server->cq, NULL, NULL, client_metadata,
-                            begin_legacy_request, tag_new);
 }
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server) {

src/core/surface/server.h

@@ -48,7 +48,7 @@ grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
    and when it shuts down, it will call destroy */
 void grpc_server_add_listener(grpc_server *server, void *listener,
                               void (*start)(grpc_server *server, void *arg,
-                                            grpc_pollset *pollset),
+                                            grpc_pollset **pollsets, size_t npollsets),
                               void (*destroy)(grpc_server *server, void *arg));
 
 /* Setup a transport - creates a channel stack, binds the transport to the

src/core/surface/server_chttp2.c

@@ -59,9 +59,9 @@ static void new_transport(void *server, grpc_endpoint *tcp) {
 }
 
 /* Server callback: start listening on our ports */
-static void start(grpc_server *server, void *tcpp, grpc_pollset *pollset) {
+static void start(grpc_server *server, void *tcpp, grpc_pollset **pollsets, size_t pollset_count) {
   grpc_tcp_server *tcp = tcpp;
-  grpc_tcp_server_start(tcp, pollset, new_transport, server);
+  grpc_tcp_server_start(tcp, pollsets, pollset_count, new_transport, server);
 }
 
 /* Server callback: destroy the tcp listener (so we don't generate further

src/core/transport/chttp2_transport.c

@@ -483,9 +483,6 @@ static void init_transport(transport *t, grpc_transport_setup_callback setup,
   ref_transport(t);
   gpr_mu_unlock(&t->mu);
 
-  ref_transport(t);
-  recv_data(t, slices, nslices, GRPC_ENDPOINT_CB_OK);
-
   sr = setup(arg, &t->base, t->metadata_context);
 
   lock(t);
@@ -494,6 +491,10 @@ static void init_transport(transport *t, grpc_transport_setup_callback setup,
   t->calling_back = 0;
   if (t->destroying) gpr_cv_signal(&t->cv);
   unlock(t);
+
+  ref_transport(t);
+  recv_data(t, slices, nslices, GRPC_ENDPOINT_CB_OK);
+
   unref_transport(t);
 }
 
@@ -1024,6 +1025,8 @@ static void cancel_stream_inner(transport *t, stream *s, gpr_uint32 id,
   int had_outgoing;
   char buffer[GPR_LTOA_MIN_BUFSIZE];
 
+  gpr_log(GPR_DEBUG, "cancel %d", id);
+
   if (s) {
     /* clear out any unreported input & output: nobody cares anymore */
     had_outgoing = s->outgoing_sopb.nops != 0;

src/cpp/client/channel.cc

@@ -42,11 +42,12 @@
 #include <grpc/support/slice.h>
 
 #include "src/cpp/proto/proto_utils.h"
-#include "src/cpp/stream/stream_context.h"
 #include <grpc++/channel_arguments.h>
 #include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
 #include <grpc++/config.h>
 #include <grpc++/credentials.h>
+#include <grpc++/impl/call.h>
 #include <grpc++/impl/rpc_method.h>
 #include <grpc++/status.h>
 #include <google/protobuf/message.h>
@@ -77,103 +78,25 @@ Channel::Channel(const grpc::string &target,
 
 Channel::~Channel() { grpc_channel_destroy(c_channel_); }
 
-namespace {
+Call Channel::CreateCall(const RpcMethod &method, ClientContext *context,
-// Pluck the finished event and set to status when it is not nullptr.
+                         CompletionQueue *cq) {
-void GetFinalStatus(grpc_completion_queue *cq, void *finished_tag,
+  auto c_call =
-                    Status *status) {
+      grpc_channel_create_call(
-  grpc_event *ev =
+          c_channel_, cq->cq(), method.name(),
-      grpc_completion_queue_pluck(cq, finished_tag, gpr_inf_future);
+          context->authority().empty() ? target_.c_str()
-  if (status) {
+                                       : context->authority().c_str(),
-    StatusCode error_code = static_cast<StatusCode>(ev->data.finished.status);
+          context->RawDeadline());
-    grpc::string details(ev->data.finished.details ? ev->data.finished.details
+  context->set_call(c_call);
-                                                   : "");
+  return Call(c_call, this, cq);
-    *status = Status(error_code, details);
-  }
-  grpc_event_finish(ev);
 }
-}  // namespace
 
-// TODO(yangg) more error handling
+void Channel::PerformOpsOnCall(CallOpBuffer *buf, Call *call) {
-Status Channel::StartBlockingRpc(const RpcMethod &method,
+  static const size_t MAX_OPS = 8;
-                                 ClientContext *context,
+  size_t nops = MAX_OPS;
-                                 const google::protobuf::Message &request,
+  grpc_op ops[MAX_OPS];
-                                 google::protobuf::Message *result) {
+  buf->FillOps(ops, &nops);
-  Status status;
+  GPR_ASSERT(GRPC_CALL_OK ==
-  grpc_call *call = grpc_channel_create_call_old(
+             grpc_call_start_batch(call->call(), ops, nops, buf));
-      c_channel_, method.name(), target_.c_str(), context->RawDeadline());
-  context->set_call(call);
-
-  grpc_event *ev;
-  void *finished_tag = reinterpret_cast<char *>(call);
-  void *metadata_read_tag = reinterpret_cast<char *>(call) + 2;
-  void *write_tag = reinterpret_cast<char *>(call) + 3;
-  void *halfclose_tag = reinterpret_cast<char *>(call) + 4;
-  void *read_tag = reinterpret_cast<char *>(call) + 5;
-
-  grpc_completion_queue *cq = grpc_completion_queue_create();
-  context->set_cq(cq);
-  // add_metadata from context
-  //
-  // invoke
-  GPR_ASSERT(grpc_call_invoke_old(call, cq, metadata_read_tag, finished_tag,
-                                  GRPC_WRITE_BUFFER_HINT) == GRPC_CALL_OK);
-  // write request
-  grpc_byte_buffer *write_buffer = nullptr;
-  bool success = SerializeProto(request, &write_buffer);
-  if (!success) {
-    grpc_call_cancel(call);
-    status =
-        Status(StatusCode::DATA_LOSS, "Failed to serialize request proto.");
-    GetFinalStatus(cq, finished_tag, nullptr);
-    return status;
-  }
-  GPR_ASSERT(grpc_call_start_write_old(call, write_buffer, write_tag,
-                                       GRPC_WRITE_BUFFER_HINT) == GRPC_CALL_OK);
-  grpc_byte_buffer_destroy(write_buffer);
-  ev = grpc_completion_queue_pluck(cq, write_tag, gpr_inf_future);
-
-  success = ev->data.write_accepted == GRPC_OP_OK;
-  grpc_event_finish(ev);
-  if (!success) {
-    GetFinalStatus(cq, finished_tag, &status);
-    return status;
-  }
-  // writes done
-  GPR_ASSERT(grpc_call_writes_done_old(call, halfclose_tag) == GRPC_CALL_OK);
-  ev = grpc_completion_queue_pluck(cq, halfclose_tag, gpr_inf_future);
-  grpc_event_finish(ev);
-  // start read metadata
-  //
-  ev = grpc_completion_queue_pluck(cq, metadata_read_tag, gpr_inf_future);
-  grpc_event_finish(ev);
-  // start read
-  GPR_ASSERT(grpc_call_start_read_old(call, read_tag) == GRPC_CALL_OK);
-  ev = grpc_completion_queue_pluck(cq, read_tag, gpr_inf_future);
-  if (ev->data.read) {
-    if (!DeserializeProto(ev->data.read, result)) {
-      grpc_event_finish(ev);
-      status = Status(StatusCode::DATA_LOSS, "Failed to parse response proto.");
-      GetFinalStatus(cq, finished_tag, nullptr);
-      return status;
-    }
-  }
-  grpc_event_finish(ev);
-
-  // wait status
-  GetFinalStatus(cq, finished_tag, &status);
-  return status;
-}
-
-StreamContextInterface *Channel::CreateStream(
-    const RpcMethod &method, ClientContext *context,
-    const google::protobuf::Message *request,
-    google::protobuf::Message *result) {
-  grpc_call *call = grpc_channel_create_call_old(
-      c_channel_, method.name(), target_.c_str(), context->RawDeadline());
-  context->set_call(call);
-  grpc_completion_queue *cq = grpc_completion_queue_create();
-  context->set_cq(cq);
-  return new StreamContext(method, context, request, result);
 }
 
 }  // namespace grpc

src/cpp/client/channel.h

@@ -42,11 +42,14 @@
 struct grpc_channel;
 
 namespace grpc {
+class Call;
+class CallOpBuffer;
 class ChannelArguments;
+class CompletionQueue;
 class Credentials;
 class StreamContextInterface;
 
-class Channel : public ChannelInterface {
+class Channel final : public ChannelInterface {
  public:
   Channel(const grpc::string &target, const ChannelArguments &args);
   Channel(const grpc::string &target, const std::unique_ptr<Credentials> &creds,
@@ -54,14 +57,9 @@ class Channel : public ChannelInterface {
 
   ~Channel() override;
 
-  Status StartBlockingRpc(const RpcMethod &method, ClientContext *context,
+  virtual Call CreateCall(const RpcMethod &method, ClientContext *context,
-                          const google::protobuf::Message &request,
+                          CompletionQueue *cq) override;
-                          google::protobuf::Message *result) override;
+  virtual void PerformOpsOnCall(CallOpBuffer *ops, Call *call) override;
-
-  StreamContextInterface *CreateStream(
-      const RpcMethod &method, ClientContext *context,
-      const google::protobuf::Message *request,
-      google::protobuf::Message *result) override;
 
  private:
   const grpc::string target_;

src/cpp/client/client_context.cc

@@ -72,9 +72,13 @@ system_clock::time_point ClientContext::absolute_deadline() {
 
 void ClientContext::AddMetadata(const grpc::string &meta_key,
                                 const grpc::string &meta_value) {
-  return;
+  send_initial_metadata_.insert(std::make_pair(meta_key, meta_value));
 }
 
-void ClientContext::StartCancel() {}
+void ClientContext::TryCancel() {
+  if (call_) {
+    grpc_call_cancel(call_);
+  }
+}
 
 }  // namespace grpc

src/cpp/client/client_unary_call.cc

@@ -0,0 +1,89 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/impl/client_unary_call.h>
+#include <grpc++/impl/call.h>
+#include <grpc++/channel_interface.h>
+#include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
+#include <grpc++/status.h>
+#include <grpc/support/log.h>
+
+namespace grpc {
+
+// Wrapper that performs a blocking unary call
+Status BlockingUnaryCall(ChannelInterface *channel, const RpcMethod &method,
+                         ClientContext *context,
+                         const google::protobuf::Message &request,
+                         google::protobuf::Message *result) {
+  CompletionQueue cq;
+  Call call(channel->CreateCall(method, context, &cq));
+  CallOpBuffer buf;
+  Status status;
+  buf.AddSendInitialMetadata(context);
+  buf.AddSendMessage(request);
+  buf.AddRecvInitialMetadata(context);
+  buf.AddRecvMessage(result);
+  buf.AddClientSendClose();
+  buf.AddClientRecvStatus(context, &status);
+  call.PerformOps(&buf);
+  GPR_ASSERT((cq.Pluck(&buf) && buf.got_message) || !status.IsOk());
+  return status;
+}
+
+class ClientAsyncRequest final : public CallOpBuffer {
+ public:
+  void FinalizeResult(void **tag, bool *status) override {
+    CallOpBuffer::FinalizeResult(tag, status);
+    delete this;
+  }
+};
+
+void AsyncUnaryCall(ChannelInterface *channel, const RpcMethod &method,
+                    ClientContext *context,
+                    const google::protobuf::Message &request,
+                    google::protobuf::Message *result, Status *status,
+                    CompletionQueue *cq, void *tag) {
+  ClientAsyncRequest *buf = new ClientAsyncRequest;
+  buf->Reset(tag);
+  Call call(channel->CreateCall(method, context, cq));
+  buf->AddSendInitialMetadata(context);
+  buf->AddSendMessage(request);
+  buf->AddRecvInitialMetadata(context);
+  buf->AddRecvMessage(result);
+  buf->AddClientSendClose();
+  buf->AddClientRecvStatus(context, status);
+  call.PerformOps(buf);
+}
+
+}  // namespace grpc

src/cpp/common/call.cc

@@ -0,0 +1,286 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <google/protobuf/message.h>
+#include <grpc/support/alloc.h>
+#include <grpc++/impl/call.h>
+#include <grpc++/client_context.h>
+#include <grpc++/channel_interface.h>
+
+#include "src/cpp/proto/proto_utils.h"
+
+namespace grpc {
+
+void CallOpBuffer::Reset(void* next_return_tag) {
+  return_tag_ = next_return_tag;
+
+  send_initial_metadata_ = false;
+  initial_metadata_count_ = 0;
+  gpr_free(initial_metadata_);
+
+  recv_initial_metadata_ = nullptr;
+  recv_initial_metadata_arr_.count = 0;
+
+  send_message_ = nullptr;
+  if (send_message_buf_) {
+    grpc_byte_buffer_destroy(send_message_buf_);
+    send_message_buf_ = nullptr;
+  }
+
+  recv_message_ = nullptr;
+  got_message = false;
+  if (recv_message_buf_) {
+    grpc_byte_buffer_destroy(recv_message_buf_);
+    recv_message_buf_ = nullptr;
+  }
+
+  client_send_close_ = false;
+
+  recv_trailing_metadata_ = nullptr;
+  recv_status_ = nullptr;
+  recv_trailing_metadata_arr_.count = 0;
+
+  status_code_ = GRPC_STATUS_OK;
+
+  send_status_ = nullptr;
+  trailing_metadata_count_ = 0;
+  trailing_metadata_ = nullptr;
+
+  recv_closed_ = nullptr;
+}
+
+CallOpBuffer::~CallOpBuffer() {
+  gpr_free(status_details_);
+  gpr_free(recv_initial_metadata_arr_.metadata);
+  gpr_free(recv_trailing_metadata_arr_.metadata);
+  if (recv_message_buf_) {
+    grpc_byte_buffer_destroy(recv_message_buf_);
+  }
+  if (send_message_buf_) {
+    grpc_byte_buffer_destroy(send_message_buf_);
+  }
+}
+
+namespace {
+// TODO(yangg) if the map is changed before we send, the pointers will be a
+// mess. Make sure it does not happen.
+grpc_metadata* FillMetadataArray(
+    std::multimap<grpc::string, grpc::string>* metadata) {
+  if (metadata->empty()) {
+    return nullptr;
+  }
+  grpc_metadata* metadata_array =
+      (grpc_metadata*)gpr_malloc(metadata->size() * sizeof(grpc_metadata));
+  size_t i = 0;
+  for (auto iter = metadata->cbegin(); iter != metadata->cend(); ++iter, ++i) {
+    metadata_array[i].key = iter->first.c_str();
+    metadata_array[i].value = iter->second.c_str();
+    metadata_array[i].value_length = iter->second.size();
+  }
+  return metadata_array;
+}
+
+void FillMetadataMap(grpc_metadata_array* arr,
+                     std::multimap<grpc::string, grpc::string>* metadata) {
+  for (size_t i = 0; i < arr->count; i++) {
+    // TODO(yangg) handle duplicates?
+    metadata->insert(std::pair<grpc::string, grpc::string>(
+        arr->metadata[i].key,
+        {arr->metadata[i].value, arr->metadata[i].value_length}));
+  }
+  grpc_metadata_array_destroy(arr);
+  grpc_metadata_array_init(arr);
+}
+}  // namespace
+
+void CallOpBuffer::AddSendInitialMetadata(
+    std::multimap<grpc::string, grpc::string>* metadata) {
+  send_initial_metadata_ = true;
+  initial_metadata_count_ = metadata->size();
+  initial_metadata_ = FillMetadataArray(metadata);
+}
+
+void CallOpBuffer::AddRecvInitialMetadata(ClientContext* ctx) {
+  ctx->initial_metadata_received_ = true;
+  recv_initial_metadata_ = &ctx->recv_initial_metadata_;
+}
+
+void CallOpBuffer::AddSendInitialMetadata(ClientContext* ctx) {
+  AddSendInitialMetadata(&ctx->send_initial_metadata_);
+}
+
+void CallOpBuffer::AddSendMessage(const google::protobuf::Message& message) {
+  send_message_ = &message;
+}
+
+void CallOpBuffer::AddRecvMessage(google::protobuf::Message* message) {
+  recv_message_ = message;
+  recv_message_->Clear();
+}
+
+void CallOpBuffer::AddClientSendClose() { client_send_close_ = true; }
+
+void CallOpBuffer::AddServerRecvClose(bool* cancelled) {
+  recv_closed_ = cancelled;
+}
+
+void CallOpBuffer::AddClientRecvStatus(ClientContext* context, Status* status) {
+  recv_trailing_metadata_ = &context->trailing_metadata_;
+  recv_status_ = status;
+}
+
+void CallOpBuffer::AddServerSendStatus(
+    std::multimap<grpc::string, grpc::string>* metadata, const Status& status) {
+  if (metadata != NULL) {
+    trailing_metadata_count_ = metadata->size();
+    trailing_metadata_ = FillMetadataArray(metadata);
+  } else {
+    trailing_metadata_count_ = 0;
+  }
+  send_status_ = &status;
+}
+
+void CallOpBuffer::FillOps(grpc_op* ops, size_t* nops) {
+  *nops = 0;
+  if (send_initial_metadata_) {
+    ops[*nops].op = GRPC_OP_SEND_INITIAL_METADATA;
+    ops[*nops].data.send_initial_metadata.count = initial_metadata_count_;
+    ops[*nops].data.send_initial_metadata.metadata = initial_metadata_;
+    (*nops)++;
+  }
+  if (recv_initial_metadata_) {
+    ops[*nops].op = GRPC_OP_RECV_INITIAL_METADATA;
+    ops[*nops].data.recv_initial_metadata = &recv_initial_metadata_arr_;
+    (*nops)++;
+  }
+  if (send_message_) {
+    bool success = SerializeProto(*send_message_, &send_message_buf_);
+    if (!success) {
+      abort();
+      // TODO handle parse failure
+    }
+    ops[*nops].op = GRPC_OP_SEND_MESSAGE;
+    ops[*nops].data.send_message = send_message_buf_;
+    (*nops)++;
+  }
+  if (recv_message_) {
+    ops[*nops].op = GRPC_OP_RECV_MESSAGE;
+    ops[*nops].data.recv_message = &recv_message_buf_;
+    (*nops)++;
+  }
+  if (client_send_close_) {
+    ops[*nops].op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+    (*nops)++;
+  }
+  if (recv_status_) {
+    ops[*nops].op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+    ops[*nops].data.recv_status_on_client.trailing_metadata =
+        &recv_trailing_metadata_arr_;
+    ops[*nops].data.recv_status_on_client.status = &status_code_;
+    ops[*nops].data.recv_status_on_client.status_details = &status_details_;
+    ops[*nops].data.recv_status_on_client.status_details_capacity =
+        &status_details_capacity_;
+    (*nops)++;
+  }
+  if (send_status_) {
+    ops[*nops].op = GRPC_OP_SEND_STATUS_FROM_SERVER;
+    ops[*nops].data.send_status_from_server.trailing_metadata_count =
+        trailing_metadata_count_;
+    ops[*nops].data.send_status_from_server.trailing_metadata =
+        trailing_metadata_;
+    ops[*nops].data.send_status_from_server.status =
+        static_cast<grpc_status_code>(send_status_->code());
+    ops[*nops].data.send_status_from_server.status_details =
+        send_status_->details().c_str();
+    (*nops)++;
+  }
+  if (recv_closed_) {
+    ops[*nops].op = GRPC_OP_RECV_CLOSE_ON_SERVER;
+    ops[*nops].data.recv_close_on_server.cancelled = &cancelled_buf_;
+    (*nops)++;
+  }
+}
+
+void CallOpBuffer::FinalizeResult(void** tag, bool* status) {
+  // Release send buffers.
+  if (send_message_buf_) {
+    grpc_byte_buffer_destroy(send_message_buf_);
+    send_message_buf_ = nullptr;
+  }
+  if (initial_metadata_) {
+    gpr_free(initial_metadata_);
+    initial_metadata_ = nullptr;
+  }
+  if (trailing_metadata_count_) {
+    gpr_free(trailing_metadata_);
+    trailing_metadata_ = nullptr;
+  }
+  // Set user-facing tag.
+  *tag = return_tag_;
+  // Process received initial metadata
+  if (recv_initial_metadata_) {
+    FillMetadataMap(&recv_initial_metadata_arr_, recv_initial_metadata_);
+  }
+  // Parse received message if any.
+  if (recv_message_) {
+    if (recv_message_buf_) {
+      got_message = *status;
+      *status = *status && DeserializeProto(recv_message_buf_, recv_message_);
+      grpc_byte_buffer_destroy(recv_message_buf_);
+      recv_message_buf_ = nullptr;
+    } else {
+      // Read failed
+      got_message = false;
+      *status = false;
+    }
+  }
+  // Parse received status.
+  if (recv_status_) {
+    FillMetadataMap(&recv_trailing_metadata_arr_, recv_trailing_metadata_);
+    *recv_status_ = Status(
+        static_cast<StatusCode>(status_code_),
+        status_details_ ? grpc::string(status_details_) : grpc::string());
+  }
+  if (recv_closed_) {
+    *recv_closed_ = cancelled_buf_ != 0;
+  }
+}
+
+Call::Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq)
+    : call_hook_(call_hook), cq_(cq), call_(call) {}
+
+void Call::PerformOps(CallOpBuffer* buffer) {
+  call_hook_->PerformOpsOnCall(buffer, this);
+}
+
+}  // namespace grpc

src/cpp/common/completion_queue.cc

@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2014, Google Inc.
  * All rights reserved.
  *
@@ -33,80 +32,54 @@
 
 #include <grpc++/completion_queue.h>
 
+#include <memory>
+
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include "src/cpp/util/time.h"
-#include <grpc++/async_server_context.h>
 
 namespace grpc {
 
 CompletionQueue::CompletionQueue() { cq_ = grpc_completion_queue_create(); }
 
+CompletionQueue::CompletionQueue(grpc_completion_queue *take) : cq_(take) {}
+
 CompletionQueue::~CompletionQueue() { grpc_completion_queue_destroy(cq_); }
 
 void CompletionQueue::Shutdown() { grpc_completion_queue_shutdown(cq_); }
 
-CompletionQueue::CompletionType CompletionQueue::Next(void **tag) {
+// Helper class so we can declare a unique_ptr with grpc_event
-  grpc_event *ev;
+class EventDeleter {
-  CompletionType return_type;
+ public:
-  bool success;
+  void operator()(grpc_event *ev) {
-
+    if (ev) grpc_event_finish(ev);
-  ev = grpc_completion_queue_next(cq_, gpr_inf_future);
-  if (!ev) {
-    gpr_log(GPR_ERROR, "no next event in queue");
-    abort();
-  }
-  switch (ev->type) {
-    case GRPC_QUEUE_SHUTDOWN:
-      return_type = QUEUE_CLOSED;
-      break;
-    case GRPC_READ:
-      *tag = ev->tag;
-      if (ev->data.read) {
-        success = static_cast<AsyncServerContext *>(ev->tag)
-                      ->ParseRead(ev->data.read);
-        return_type = success ? SERVER_READ_OK : SERVER_READ_ERROR;
-      } else {
-        return_type = SERVER_READ_ERROR;
-      }
-      break;
-    case GRPC_WRITE_ACCEPTED:
-      *tag = ev->tag;
-      if (ev->data.write_accepted != GRPC_OP_ERROR) {
-        return_type = SERVER_WRITE_OK;
-      } else {
-        return_type = SERVER_WRITE_ERROR;
   }
-      break;
+};
-    case GRPC_SERVER_RPC_NEW:
+
-      GPR_ASSERT(!ev->tag);
+bool CompletionQueue::Next(void **tag, bool *ok) {
-      // Finishing the pending new rpcs after the server has been shutdown.
+  std::unique_ptr<grpc_event, EventDeleter> ev;
-      if (!ev->call) {
+
-        *tag = nullptr;
+  ev.reset(grpc_completion_queue_next(cq_, gpr_inf_future));
-      } else {
+  if (ev->type == GRPC_QUEUE_SHUTDOWN) {
-        *tag = new AsyncServerContext(
+    return false;
-            ev->call, ev->data.server_rpc_new.method,
-            ev->data.server_rpc_new.host,
-            Timespec2Timepoint(ev->data.server_rpc_new.deadline));
-      }
-      return_type = SERVER_RPC_NEW;
-      break;
-    case GRPC_FINISHED:
-      *tag = ev->tag;
-      return_type = RPC_END;
-      break;
-    case GRPC_FINISH_ACCEPTED:
-      *tag = ev->tag;
-      return_type = HALFCLOSE_OK;
-      break;
-    default:
-      // We do not handle client side messages now
-      gpr_log(GPR_ERROR, "client-side messages aren't supported yet");
-      abort();
   }
-  grpc_event_finish(ev);
+  auto cq_tag = static_cast<CompletionQueueTag *>(ev->tag);
-  return return_type;
+  *ok = ev->data.op_complete == GRPC_OP_OK;
+  *tag = cq_tag;
+  cq_tag->FinalizeResult(tag, ok);
+  return true;
+}
+
+bool CompletionQueue::Pluck(CompletionQueueTag *tag) {
+  std::unique_ptr<grpc_event, EventDeleter> ev;
+
+  ev.reset(grpc_completion_queue_pluck(cq_, tag, gpr_inf_future));
+  bool ok = ev->data.op_complete == GRPC_OP_OK;
+  void *ignored = tag;
+  tag->FinalizeResult(&ignored, &ok);
+  GPR_ASSERT(ignored == tag);
+  return ok;
 }
 
 }  // namespace grpc

src/cpp/server/async_server.cc

@@ -1,89 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <grpc++/async_server.h>
-
-#include <grpc/grpc.h>
-#include <grpc/support/log.h>
-#include <grpc++/completion_queue.h>
-
-namespace grpc {
-
-AsyncServer::AsyncServer(CompletionQueue *cc)
-    : started_(false), shutdown_(false) {
-  server_ = grpc_server_create(cc->cq(), nullptr);
-}
-
-AsyncServer::~AsyncServer() {
-  std::unique_lock<std::mutex> lock(shutdown_mu_);
-  if (started_ && !shutdown_) {
-    lock.unlock();
-    Shutdown();
-  }
-  grpc_server_destroy(server_);
-}
-
-void AsyncServer::AddPort(const grpc::string &addr) {
-  GPR_ASSERT(!started_);
-  int success = grpc_server_add_http2_port(server_, addr.c_str());
-  GPR_ASSERT(success);
-}
-
-void AsyncServer::Start() {
-  GPR_ASSERT(!started_);
-  started_ = true;
-  grpc_server_start(server_);
-}
-
-void AsyncServer::RequestOneRpc() {
-  GPR_ASSERT(started_);
-  std::unique_lock<std::mutex> lock(shutdown_mu_);
-  if (shutdown_) {
-    return;
-  }
-  lock.unlock();
-  grpc_call_error err = grpc_server_request_call_old(server_, nullptr);
-  GPR_ASSERT(err == GRPC_CALL_OK);
-}
-
-void AsyncServer::Shutdown() {
-  std::unique_lock<std::mutex> lock(shutdown_mu_);
-  if (started_ && !shutdown_) {
-    shutdown_ = true;
-    lock.unlock();
-    // TODO(yangg) should we shutdown without start?
-    grpc_server_shutdown(server_);
-  }
-}
-
-}  // namespace grpc

src/cpp/server/async_server_context.cc

@@ -54,8 +54,8 @@ AsyncServerContext::~AsyncServerContext() { grpc_call_destroy(call_); }
 
 void AsyncServerContext::Accept(grpc_completion_queue *cq) {
   GPR_ASSERT(grpc_call_server_accept_old(call_, cq, this) == GRPC_CALL_OK);
-  GPR_ASSERT(grpc_call_server_end_initial_metadata_old(call_, GRPC_WRITE_BUFFER_HINT) ==
+  GPR_ASSERT(grpc_call_server_end_initial_metadata_old(
-             GRPC_CALL_OK);
+                 call_, GRPC_WRITE_BUFFER_HINT) == GRPC_CALL_OK);
 }
 
 bool AsyncServerContext::StartRead(google::protobuf::Message *request) {

src/cpp/server/server.cc

@@ -37,25 +37,25 @@
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
 #include <grpc/support/log.h>
-#include "src/cpp/server/server_rpc_handler.h"
-#include "src/cpp/server/thread_pool.h"
-#include <grpc++/async_server_context.h>
 #include <grpc++/completion_queue.h>
 #include <grpc++/impl/rpc_service_method.h>
+#include <grpc++/impl/service_type.h>
+#include <grpc++/server_context.h>
 #include <grpc++/server_credentials.h>
+#include <grpc++/thread_pool_interface.h>
 
-namespace grpc {
+#include "src/cpp/proto/proto_utils.h"
+#include "src/cpp/util/time.h"
 
-// TODO(rocking): consider a better default value like num of cores.
+namespace grpc {
-static const int kNumThreads = 4;
 
-Server::Server(ThreadPoolInterface *thread_pool, ServerCredentials *creds)
+Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
+               ServerCredentials* creds)
     : started_(false),
       shutdown_(false),
       num_running_cb_(0),
-      thread_pool_(thread_pool == nullptr ? new ThreadPool(kNumThreads)
+      thread_pool_(thread_pool),
-                                          : thread_pool),
+      thread_pool_owned_(thread_pool_owned),
-      thread_pool_owned_(thread_pool == nullptr),
       secure_(creds != nullptr) {
   if (creds) {
     server_ =
@@ -75,6 +75,8 @@ Server::~Server() {
   if (started_ && !shutdown_) {
     lock.unlock();
     Shutdown();
+  } else {
+    lock.unlock();
   }
   grpc_server_destroy(server_);
   if (thread_pool_owned_) {
@@ -82,37 +84,180 @@ Server::~Server() {
   }
 }
 
-void Server::RegisterService(RpcService *service) {
+bool Server::RegisterService(RpcService* service) {
   for (int i = 0; i < service->GetMethodCount(); ++i) {
-    RpcServiceMethod *method = service->GetMethod(i);
+    RpcServiceMethod* method = service->GetMethod(i);
-    method_map_.insert(std::make_pair(method->name(), method));
+    void* tag =
+        grpc_server_register_method(server_, method->name(), nullptr, cq_.cq());
+    if (!tag) {
+      gpr_log(GPR_DEBUG, "Attempt to register %s multiple times",
+              method->name());
+      return false;
+    }
+    sync_methods_.emplace_back(method, tag);
+  }
+  return true;
+}
+
+bool Server::RegisterAsyncService(AsynchronousService* service) {
+  GPR_ASSERT(service->dispatch_impl_ == nullptr &&
+             "Can only register an asynchronous service against one server.");
+  service->dispatch_impl_ = this;
+  service->request_args_ = new void* [service->method_count_];
+  for (size_t i = 0; i < service->method_count_; ++i) {
+    void* tag =
+        grpc_server_register_method(server_, service->method_names_[i], nullptr,
+                                    service->completion_queue()->cq());
+    if (!tag) {
+      gpr_log(GPR_DEBUG, "Attempt to register %s multiple times",
+              service->method_names_[i]);
+      return false;
+    }
+    service->request_args_[i] = tag;
   }
+  return true;
 }
 
-void Server::AddPort(const grpc::string &addr) {
+int Server::AddPort(const grpc::string& addr) {
   GPR_ASSERT(!started_);
-  int success;
   if (secure_) {
-    success = grpc_server_add_secure_http2_port(server_, addr.c_str());
+    return grpc_server_add_secure_http2_port(server_, addr.c_str());
   } else {
-    success = grpc_server_add_http2_port(server_, addr.c_str());
+    return grpc_server_add_http2_port(server_, addr.c_str());
   }
-  GPR_ASSERT(success);
 }
 
-void Server::Start() {
+class Server::SyncRequest final : public CompletionQueueTag {
+ public:
+  SyncRequest(RpcServiceMethod* method, void* tag)
+      : method_(method),
+        tag_(tag),
+        has_request_payload_(method->method_type() == RpcMethod::NORMAL_RPC ||
+                             method->method_type() ==
+                                 RpcMethod::SERVER_STREAMING),
+        has_response_payload_(method->method_type() == RpcMethod::NORMAL_RPC ||
+                              method->method_type() ==
+                                  RpcMethod::CLIENT_STREAMING) {
+    grpc_metadata_array_init(&request_metadata_);
+  }
+
+  static SyncRequest* Wait(CompletionQueue* cq, bool* ok) {
+    void* tag = nullptr;
+    *ok = false;
+    if (!cq->Next(&tag, ok)) {
+      return nullptr;
+    }
+    auto* mrd = static_cast<SyncRequest*>(tag);
+    GPR_ASSERT(mrd->in_flight_);
+    return mrd;
+  }
+
+  void Request(grpc_server* server) {
+    GPR_ASSERT(!in_flight_);
+    in_flight_ = true;
+    cq_ = grpc_completion_queue_create();
+    GPR_ASSERT(GRPC_CALL_OK ==
+               grpc_server_request_registered_call(
+                   server, tag_, &call_, &deadline_, &request_metadata_,
+                   has_request_payload_ ? &request_payload_ : nullptr, cq_,
+                   this));
+  }
+
+  void FinalizeResult(void** tag, bool* status) override {
+    if (!*status) {
+      grpc_completion_queue_destroy(cq_);
+    }
+  }
+
+  class CallData final {
+   public:
+    explicit CallData(Server* server, SyncRequest* mrd)
+        : cq_(mrd->cq_),
+          call_(mrd->call_, server, &cq_),
+          ctx_(mrd->deadline_, mrd->request_metadata_.metadata,
+               mrd->request_metadata_.count),
+          has_request_payload_(mrd->has_request_payload_),
+          has_response_payload_(mrd->has_response_payload_),
+          request_payload_(mrd->request_payload_),
+          method_(mrd->method_) {
+      ctx_.call_ = mrd->call_;
+      GPR_ASSERT(mrd->in_flight_);
+      mrd->in_flight_ = false;
+      mrd->request_metadata_.count = 0;
+    }
+
+    ~CallData() {
+      if (has_request_payload_ && request_payload_) {
+        grpc_byte_buffer_destroy(request_payload_);
+      }
+    }
+
+    void Run() {
+      std::unique_ptr<google::protobuf::Message> req;
+      std::unique_ptr<google::protobuf::Message> res;
+      if (has_request_payload_) {
+        req.reset(method_->AllocateRequestProto());
+        if (!DeserializeProto(request_payload_, req.get())) {
+          abort();  // for now
+        }
+      }
+      if (has_response_payload_) {
+        res.reset(method_->AllocateResponseProto());
+      }
+      auto status = method_->handler()->RunHandler(
+          MethodHandler::HandlerParameter(&call_, &ctx_, req.get(), res.get()));
+      CallOpBuffer buf;
+      if (!ctx_.sent_initial_metadata_) {
+        buf.AddSendInitialMetadata(&ctx_.initial_metadata_);
+      }
+      if (has_response_payload_) {
+        buf.AddSendMessage(*res);
+      }
+      buf.AddServerSendStatus(&ctx_.trailing_metadata_, status);
+      bool cancelled;
+      buf.AddServerRecvClose(&cancelled);
+      call_.PerformOps(&buf);
+      GPR_ASSERT(cq_.Pluck(&buf));
+    }
+
+   private:
+    CompletionQueue cq_;
+    Call call_;
+    ServerContext ctx_;
+    const bool has_request_payload_;
+    const bool has_response_payload_;
+    grpc_byte_buffer* request_payload_;
+    RpcServiceMethod* const method_;
+  };
+
+ private:
+  RpcServiceMethod* const method_;
+  void* const tag_;
+  bool in_flight_ = false;
+  const bool has_request_payload_;
+  const bool has_response_payload_;
+  grpc_call* call_;
+  gpr_timespec deadline_;
+  grpc_metadata_array request_metadata_;
+  grpc_byte_buffer* request_payload_;
+  grpc_completion_queue* cq_;
+};
+
+bool Server::Start() {
   GPR_ASSERT(!started_);
   started_ = true;
   grpc_server_start(server_);
 
   // Start processing rpcs.
+  if (!sync_methods_.empty()) {
+    for (auto& m : sync_methods_) {
+      m.Request(server_);
+    }
+
     ScheduleCallback();
-}
+  }
 
-void Server::AllowOneRpc() {
+  return true;
-  GPR_ASSERT(started_);
-  grpc_call_error err = grpc_server_request_call_old(server_, nullptr);
-  GPR_ASSERT(err == GRPC_CALL_OK);
 }
 
 void Server::Shutdown() {
@@ -121,6 +266,7 @@ void Server::Shutdown() {
     if (started_ && !shutdown_) {
       shutdown_ = true;
       grpc_server_shutdown(server_);
+      cq_.Shutdown();
 
       // Wait for running callbacks to finish.
       while (num_running_cb_ != 0) {
@@ -128,12 +274,85 @@ void Server::Shutdown() {
       }
     }
   }
+}
 
-  // Shutdown the completion queue.
+void Server::PerformOpsOnCall(CallOpBuffer* buf, Call* call) {
-  cq_.Shutdown();
+  static const size_t MAX_OPS = 8;
-  void *tag = nullptr;
+  size_t nops = MAX_OPS;
-  CompletionQueue::CompletionType t = cq_.Next(&tag);
+  grpc_op ops[MAX_OPS];
-  GPR_ASSERT(t == CompletionQueue::QUEUE_CLOSED);
+  buf->FillOps(ops, &nops);
+  GPR_ASSERT(GRPC_CALL_OK ==
+             grpc_call_start_batch(call->call(), ops, nops, buf));
+}
+
+class Server::AsyncRequest final : public CompletionQueueTag {
+ public:
+  AsyncRequest(Server* server, void* registered_method, ServerContext* ctx,
+               ::google::protobuf::Message* request,
+               ServerAsyncStreamingInterface* stream, CompletionQueue* cq,
+               void* tag)
+      : tag_(tag),
+        request_(request),
+        stream_(stream),
+        cq_(cq),
+        ctx_(ctx),
+        server_(server) {
+    memset(&array_, 0, sizeof(array_));
+    grpc_server_request_registered_call(
+        server->server_, registered_method, &call_, &deadline_, &array_,
+        request ? &payload_ : nullptr, cq->cq(), this);
+  }
+
+  ~AsyncRequest() {
+    if (payload_) {
+      grpc_byte_buffer_destroy(payload_);
+    }
+    grpc_metadata_array_destroy(&array_);
+  }
+
+  void FinalizeResult(void** tag, bool* status) override {
+    *tag = tag_;
+    if (*status && request_) {
+      if (payload_) {
+        *status = *status && DeserializeProto(payload_, request_);
+      } else {
+        *status = false;
+      }
+    }
+    if (*status) {
+      ctx_->deadline_ = Timespec2Timepoint(deadline_);
+      for (size_t i = 0; i < array_.count; i++) {
+        ctx_->client_metadata_.insert(std::make_pair(
+            grpc::string(array_.metadata[i].key),
+            grpc::string(
+                array_.metadata[i].value,
+                array_.metadata[i].value + array_.metadata[i].value_length)));
+      }
+    }
+    ctx_->call_ = call_;
+    Call call(call_, server_, cq_);
+    stream_->BindCall(&call);
+    delete this;
+  }
+
+ private:
+  void* const tag_;
+  ::google::protobuf::Message* const request_;
+  ServerAsyncStreamingInterface* const stream_;
+  CompletionQueue* const cq_;
+  ServerContext* const ctx_;
+  Server* const server_;
+  grpc_call* call_ = nullptr;
+  gpr_timespec deadline_;
+  grpc_metadata_array array_;
+  grpc_byte_buffer* payload_ = nullptr;
+};
+
+void Server::RequestAsyncCall(void* registered_method, ServerContext* context,
+                              ::google::protobuf::Message* request,
+                              ServerAsyncStreamingInterface* stream,
+                              CompletionQueue* cq, void* tag) {
+  new AsyncRequest(this, registered_method, context, request, stream, cq, tag);
 }
 
 void Server::ScheduleCallback() {
@@ -141,30 +360,21 @@ void Server::ScheduleCallback() {
     std::unique_lock<std::mutex> lock(mu_);
     num_running_cb_++;
   }
-  std::function<void()> callback = std::bind(&Server::RunRpc, this);
+  thread_pool_->ScheduleCallback(std::bind(&Server::RunRpc, this));
-  thread_pool_->ScheduleCallback(callback);
 }
 
 void Server::RunRpc() {
   // Wait for one more incoming rpc.
-  void *tag = nullptr;
+  bool ok;
-  AllowOneRpc();
+  auto* mrd = SyncRequest::Wait(&cq_, &ok);
-  CompletionQueue::CompletionType t = cq_.Next(&tag);
+  if (mrd) {
-  GPR_ASSERT(t == CompletionQueue::SERVER_RPC_NEW);
-
-  AsyncServerContext *server_context = static_cast<AsyncServerContext *>(tag);
-  // server_context could be nullptr during server shutdown.
-  if (server_context != nullptr) {
-    // Schedule a new callback to handle more rpcs.
     ScheduleCallback();
+    if (ok) {
+      SyncRequest::CallData cd(this, mrd);
+      mrd->Request(server_);
 
-    RpcServiceMethod *method = nullptr;
+      cd.Run();
-    auto iter = method_map_.find(server_context->method());
-    if (iter != method_map_.end()) {
-      method = iter->second;
     }
-    ServerRpcHandler rpc_handler(server_context, method);
-    rpc_handler.StartRpc();
   }
 
   {

src/cpp/server/server_builder.cc

@@ -33,40 +33,70 @@
 
 #include <grpc++/server_builder.h>
 
+#include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
+#include <grpc++/impl/service_type.h>
 #include <grpc++/server.h>
+#include "src/cpp/server/thread_pool.h"
 
 namespace grpc {
 
-ServerBuilder::ServerBuilder() : thread_pool_(nullptr) {}
+ServerBuilder::ServerBuilder() {}
 
-void ServerBuilder::RegisterService(RpcService *service) {
+void ServerBuilder::RegisterService(SynchronousService* service) {
-  services_.push_back(service);
+  services_.push_back(service->service());
 }
 
-void ServerBuilder::AddPort(const grpc::string &addr) {
+void ServerBuilder::RegisterAsyncService(AsynchronousService* service) {
+  async_services_.push_back(service);
+}
+
+void ServerBuilder::AddPort(const grpc::string& addr) {
   ports_.push_back(addr);
 }
 
 void ServerBuilder::SetCredentials(
-    const std::shared_ptr<ServerCredentials> &creds) {
+    const std::shared_ptr<ServerCredentials>& creds) {
   GPR_ASSERT(!creds_);
   creds_ = creds;
 }
 
-void ServerBuilder::SetThreadPool(ThreadPoolInterface *thread_pool) {
+void ServerBuilder::SetThreadPool(ThreadPoolInterface* thread_pool) {
   thread_pool_ = thread_pool;
 }
 
 std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
-  std::unique_ptr<Server> server(new Server(thread_pool_, creds_.get()));
+  bool thread_pool_owned = false;
-  for (auto *service : services_) {
+  if (!async_services_.empty() && !services_.empty()) {
-    server->RegisterService(service);
+    gpr_log(GPR_ERROR, "Mixing async and sync services is unsupported for now");
+    return nullptr;
+  }
+  if (!thread_pool_ && services_.size()) {
+    int cores = gpr_cpu_num_cores();
+    if (!cores) cores = 4;
+    thread_pool_ = new ThreadPool(cores);
+    thread_pool_owned = true;
+  }
+  std::unique_ptr<Server> server(
+      new Server(thread_pool_, thread_pool_owned, creds_.get()));
+  for (auto* service : services_) {
+    if (!server->RegisterService(service)) {
+      return nullptr;
+    }
+  }
+  for (auto* service : async_services_) {
+    if (!server->RegisterAsyncService(service)) {
+      return nullptr;
+    }
+  }
+  for (auto& port : ports_) {
+    if (!server->AddPort(port)) {
+      return nullptr;
+    }
   }
-  for (auto &port : ports_) {
+  if (!server->Start()) {
-    server->AddPort(port);
+    return nullptr;
   }
-  server->Start();
   return server;
 }
 

src/cpp/server/server_context.cc

@@ -31,31 +31,40 @@
  *
  */
 
-#ifndef __GRPCPP_INTERNAL_SERVER_SERVER_CONTEXT_IMPL_H_
-#define __GRPCPP_INTERNAL_SERVER_SERVER_CONTEXT_IMPL_H_
-
 #include <grpc++/server_context.h>
-
+#include <grpc++/impl/call.h>
-#include <chrono>
+#include <grpc/grpc.h>
-
+#include "src/cpp/util/time.h"
-#include <grpc/support/time.h>
 
 namespace grpc {
 
-class ServerContextImpl : public ServerContext {
+ServerContext::ServerContext() {}
- public:
-  explicit ServerContextImpl(std::chrono::system_clock::time_point deadline)
-      : absolute_deadline_(deadline) {}
-  ~ServerContextImpl() {}
 
-  std::chrono::system_clock::time_point absolute_deadline() const {
+ServerContext::ServerContext(gpr_timespec deadline, grpc_metadata *metadata,
-    return absolute_deadline_;
+                             size_t metadata_count)
+    : deadline_(Timespec2Timepoint(deadline)) {
+  for (size_t i = 0; i < metadata_count; i++) {
+    client_metadata_.insert(std::make_pair(
+        grpc::string(metadata[i].key),
+        grpc::string(metadata[i].value,
+                     metadata[i].value + metadata[i].value_length)));
   }
+}
 
- private:
+ServerContext::~ServerContext() {
-  std::chrono::system_clock::time_point absolute_deadline_;
+  if (call_) {
-};
+    grpc_call_destroy(call_);
+  }
+}
 
-}  // namespace grpc
+void ServerContext::AddInitialMetadata(const grpc::string& key,
+                                  const grpc::string& value) {
+  initial_metadata_.insert(std::make_pair(key, value));
+}
+
+void ServerContext::AddTrailingMetadata(const grpc::string& key,
+                                  const grpc::string& value) {
+  trailing_metadata_.insert(std::make_pair(key, value));
+}
 
-#endif  // __GRPCPP_INTERNAL_SERVER_SERVER_CONTEXT_IMPL_H_
+}  // namespace grpc

src/cpp/server/server_context_impl.cc

@@ -1,36 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/cpp/server/server_context_impl.h"
-
-namespace grpc {}  // namespace grpc

src/cpp/server/server_rpc_handler.cc

@@ -1,140 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/cpp/server/server_rpc_handler.h"
-
-#include <grpc/support/log.h>
-#include "src/cpp/server/server_context_impl.h"
-#include "src/cpp/stream/stream_context.h"
-#include <grpc++/async_server_context.h>
-#include <grpc++/impl/rpc_service_method.h>
-
-namespace grpc {
-
-ServerRpcHandler::ServerRpcHandler(AsyncServerContext *async_server_context,
-                                   RpcServiceMethod *method)
-    : async_server_context_(async_server_context), method_(method) {}
-
-void ServerRpcHandler::StartRpc() {
-  if (method_ == nullptr) {
-    // Method not supported, finish the rpc with error.
-    // TODO(rocking): do we need to call read to consume the request?
-    FinishRpc(Status(StatusCode::UNIMPLEMENTED, "No such method."));
-    return;
-  }
-
-  ServerContextImpl user_context(async_server_context_->absolute_deadline());
-
-  if (method_->method_type() == RpcMethod::NORMAL_RPC) {
-    // Start the rpc on this dedicated completion queue.
-    async_server_context_->Accept(cq_.cq());
-
-    // Allocate request and response.
-    std::unique_ptr<google::protobuf::Message> request(
-        method_->AllocateRequestProto());
-    std::unique_ptr<google::protobuf::Message> response(
-        method_->AllocateResponseProto());
-
-    // Read request
-    async_server_context_->StartRead(request.get());
-    auto type = WaitForNextEvent();
-    GPR_ASSERT(type == CompletionQueue::SERVER_READ_OK);
-
-    // Run the application's rpc handler
-    MethodHandler *handler = method_->handler();
-    Status status = handler->RunHandler(MethodHandler::HandlerParameter(
-        &user_context, request.get(), response.get()));
-
-    if (status.IsOk()) {
-      // Send the response if we get an ok status.
-      async_server_context_->StartWrite(*response, GRPC_WRITE_BUFFER_HINT);
-      type = WaitForNextEvent();
-      if (type != CompletionQueue::SERVER_WRITE_OK) {
-        status = Status(StatusCode::INTERNAL, "Error writing response.");
-      }
-    }
-
-    FinishRpc(status);
-  } else {
-    // Allocate request and response.
-    // TODO(yangg) maybe not allocate both when not needed?
-    std::unique_ptr<google::protobuf::Message> request(
-        method_->AllocateRequestProto());
-    std::unique_ptr<google::protobuf::Message> response(
-        method_->AllocateResponseProto());
-
-    StreamContext stream_context(*method_, async_server_context_->call(),
-                                 cq_.cq(), request.get(), response.get());
-
-    // Run the application's rpc handler
-    MethodHandler *handler = method_->handler();
-    Status status = handler->RunHandler(MethodHandler::HandlerParameter(
-        &user_context, request.get(), response.get(), &stream_context));
-    if (status.IsOk() &&
-        method_->method_type() == RpcMethod::CLIENT_STREAMING) {
-      stream_context.Write(response.get(), false);
-    }
-    // TODO(yangg) Do we need to consider the status in stream_context?
-    FinishRpc(status);
-  }
-}
-
-CompletionQueue::CompletionType ServerRpcHandler::WaitForNextEvent() {
-  void *tag = nullptr;
-  CompletionQueue::CompletionType type = cq_.Next(&tag);
-  if (type != CompletionQueue::QUEUE_CLOSED &&
-      type != CompletionQueue::RPC_END) {
-    GPR_ASSERT(static_cast<AsyncServerContext *>(tag) ==
-               async_server_context_.get());
-  }
-  return type;
-}
-
-void ServerRpcHandler::FinishRpc(const Status &status) {
-  async_server_context_->StartWriteStatus(status);
-  CompletionQueue::CompletionType type;
-
-  // HALFCLOSE_OK and RPC_END events come in either order.
-  type = WaitForNextEvent();
-  GPR_ASSERT(type == CompletionQueue::HALFCLOSE_OK ||
-             type == CompletionQueue::RPC_END);
-  type = WaitForNextEvent();
-  GPR_ASSERT(type == CompletionQueue::HALFCLOSE_OK ||
-             type == CompletionQueue::RPC_END);
-
-  cq_.Shutdown();
-  type = WaitForNextEvent();
-  GPR_ASSERT(type == CompletionQueue::QUEUE_CLOSED);
-}
-
-}  // namespace grpc

src/cpp/server/server_rpc_handler.h

@@ -1,66 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef __GRPCPP_INTERNAL_SERVER_SERVER_RPC_HANDLER_H__
-#define __GRPCPP_INTERNAL_SERVER_SERVER_RPC_HANDLER_H__
-
-#include <memory>
-
-#include <grpc++/completion_queue.h>
-#include <grpc++/status.h>
-
-namespace grpc {
-
-class AsyncServerContext;
-class RpcServiceMethod;
-
-class ServerRpcHandler {
- public:
-  // Takes ownership of async_server_context.
-  ServerRpcHandler(AsyncServerContext *async_server_context,
-                   RpcServiceMethod *method);
-
-  void StartRpc();
-
- private:
-  CompletionQueue::CompletionType WaitForNextEvent();
-  void FinishRpc(const Status &status);
-
-  std::unique_ptr<AsyncServerContext> async_server_context_;
-  RpcServiceMethod *method_;
-  CompletionQueue cq_;
-};
-
-}  // namespace grpc
-
-#endif  // __GRPCPP_INTERNAL_SERVER_SERVER_RPC_HANDLER_H__

src/cpp/server/thread_pool.h

@@ -44,12 +44,12 @@
 
 namespace grpc {
 
-class ThreadPool : public ThreadPoolInterface {
+class ThreadPool final : public ThreadPoolInterface {
  public:
   explicit ThreadPool(int num_threads);
   ~ThreadPool();
 
-  void ScheduleCallback(const std::function<void()> &callback) final;
+  void ScheduleCallback(const std::function<void()> &callback) override;
 
  private:
   std::mutex mu_;

src/cpp/stream/stream_context.cc

@@ -1,179 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/cpp/stream/stream_context.h"
-
-#include <grpc/support/log.h>
-#include "src/cpp/proto/proto_utils.h"
-#include "src/cpp/util/time.h"
-#include <grpc++/client_context.h>
-#include <grpc++/config.h>
-#include <grpc++/impl/rpc_method.h>
-#include <google/protobuf/message.h>
-
-namespace grpc {
-
-// Client only ctor
-StreamContext::StreamContext(const RpcMethod &method, ClientContext *context,
-                             const google::protobuf::Message *request,
-                             google::protobuf::Message *result)
-    : is_client_(true),
-      method_(&method),
-      call_(context->call()),
-      cq_(context->cq()),
-      request_(const_cast<google::protobuf::Message *>(request)),
-      result_(result),
-      peer_halfclosed_(false),
-      self_halfclosed_(false) {
-  GPR_ASSERT(method_->method_type() != RpcMethod::RpcType::NORMAL_RPC);
-}
-
-// Server only ctor
-StreamContext::StreamContext(const RpcMethod &method, grpc_call *call,
-                             grpc_completion_queue *cq,
-                             google::protobuf::Message *request,
-                             google::protobuf::Message *result)
-    : is_client_(false),
-      method_(&method),
-      call_(call),
-      cq_(cq),
-      request_(request),
-      result_(result),
-      peer_halfclosed_(false),
-      self_halfclosed_(false) {
-  GPR_ASSERT(method_->method_type() != RpcMethod::RpcType::NORMAL_RPC);
-}
-
-StreamContext::~StreamContext() {}
-
-void StreamContext::Start(bool buffered) {
-  if (is_client_) {
-    // TODO(yangg) handle metadata send path
-    int flag = buffered ? GRPC_WRITE_BUFFER_HINT : 0;
-    grpc_call_error error = grpc_call_invoke_old(
-        call(), cq(), client_metadata_read_tag(), finished_tag(), flag);
-    GPR_ASSERT(GRPC_CALL_OK == error);
-  } else {
-    // TODO(yangg) metadata needs to be added before accept
-    // TODO(yangg) correctly set flag to accept
-    GPR_ASSERT(grpc_call_server_accept_old(call(), cq(), finished_tag()) ==
-               GRPC_CALL_OK);
-    GPR_ASSERT(grpc_call_server_end_initial_metadata_old(call(), 0) ==
-               GRPC_CALL_OK);
-  }
-}
-
-bool StreamContext::Read(google::protobuf::Message *msg) {
-  // TODO(yangg) check peer_halfclosed_ here for possible early return.
-  grpc_call_error err = grpc_call_start_read_old(call(), read_tag());
-  GPR_ASSERT(err == GRPC_CALL_OK);
-  grpc_event *read_ev =
-      grpc_completion_queue_pluck(cq(), read_tag(), gpr_inf_future);
-  GPR_ASSERT(read_ev->type == GRPC_READ);
-  bool ret = true;
-  if (read_ev->data.read) {
-    if (!DeserializeProto(read_ev->data.read, msg)) {
-      ret = false;
-      grpc_call_cancel_with_status(call(), GRPC_STATUS_DATA_LOSS,
-                                   "Failed to parse incoming proto");
-    }
-  } else {
-    ret = false;
-    peer_halfclosed_ = true;
-  }
-  grpc_event_finish(read_ev);
-  return ret;
-}
-
-bool StreamContext::Write(const google::protobuf::Message *msg, bool is_last) {
-  // TODO(yangg) check self_halfclosed_ for possible early return.
-  bool ret = true;
-  grpc_event *ev = nullptr;
-
-  if (msg) {
-    grpc_byte_buffer *out_buf = nullptr;
-    if (!SerializeProto(*msg, &out_buf)) {
-      grpc_call_cancel_with_status(call(), GRPC_STATUS_INVALID_ARGUMENT,
-                                   "Failed to serialize outgoing proto");
-      return false;
-    }
-    int flag = is_last ? GRPC_WRITE_BUFFER_HINT : 0;
-    grpc_call_error err =
-        grpc_call_start_write_old(call(), out_buf, write_tag(), flag);
-    grpc_byte_buffer_destroy(out_buf);
-    GPR_ASSERT(err == GRPC_CALL_OK);
-
-    ev = grpc_completion_queue_pluck(cq(), write_tag(), gpr_inf_future);
-    GPR_ASSERT(ev->type == GRPC_WRITE_ACCEPTED);
-
-    ret = ev->data.write_accepted == GRPC_OP_OK;
-    grpc_event_finish(ev);
-  }
-  if (ret && is_last) {
-    grpc_call_error err = grpc_call_writes_done_old(call(), halfclose_tag());
-    GPR_ASSERT(err == GRPC_CALL_OK);
-    ev = grpc_completion_queue_pluck(cq(), halfclose_tag(), gpr_inf_future);
-    GPR_ASSERT(ev->type == GRPC_FINISH_ACCEPTED);
-    grpc_event_finish(ev);
-
-    self_halfclosed_ = true;
-  } else if (!ret) {  // Stream broken
-    self_halfclosed_ = true;
-    peer_halfclosed_ = true;
-  }
-
-  return ret;
-}
-
-const Status &StreamContext::Wait() {
-  // TODO(yangg) properly support metadata
-  grpc_event *metadata_ev = grpc_completion_queue_pluck(
-      cq(), client_metadata_read_tag(), gpr_inf_future);
-  grpc_event_finish(metadata_ev);
-  // TODO(yangg) protect states by a mutex, including other places.
-  if (!self_halfclosed_ || !peer_halfclosed_) {
-    Cancel();
-  }
-  grpc_event *finish_ev =
-      grpc_completion_queue_pluck(cq(), finished_tag(), gpr_inf_future);
-  GPR_ASSERT(finish_ev->type == GRPC_FINISHED);
-  final_status_ = Status(
-      static_cast<StatusCode>(finish_ev->data.finished.status),
-      finish_ev->data.finished.details ? finish_ev->data.finished.details : "");
-  grpc_event_finish(finish_ev);
-  return final_status_;
-}
-
-void StreamContext::Cancel() { grpc_call_cancel(call()); }
-
-}  // namespace grpc

src/cpp/stream/stream_context.h

@@ -1,99 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef __GRPCPP_INTERNAL_STREAM_STREAM_CONTEXT_H__
-#define __GRPCPP_INTERNAL_STREAM_STREAM_CONTEXT_H__
-
-#include <grpc/grpc.h>
-#include <grpc++/status.h>
-#include <grpc++/stream_context_interface.h>
-
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
-
-namespace grpc {
-class ClientContext;
-class RpcMethod;
-
-class StreamContext final : public StreamContextInterface {
- public:
-  StreamContext(const RpcMethod &method, ClientContext *context,
-                const google::protobuf::Message *request,
-                google::protobuf::Message *result);
-  StreamContext(const RpcMethod &method, grpc_call *call,
-                grpc_completion_queue *cq, google::protobuf::Message *request,
-                google::protobuf::Message *result);
-  ~StreamContext();
-  // Start the stream, if there is a final write following immediately, set
-  // buffered so that the messages can be sent in batch.
-  void Start(bool buffered) override;
-  bool Read(google::protobuf::Message *msg) override;
-  bool Write(const google::protobuf::Message *msg, bool is_last) override;
-  const Status &Wait() override;
-  void Cancel() override;
-
-  google::protobuf::Message *request() override { return request_; }
-  google::protobuf::Message *response() override { return result_; }
-
- private:
-  // Unique tags for plucking events from the c layer. this pointer is casted
-  // to char* to create single byte step between tags. It implicitly relies on
-  // that StreamContext is large enough to contain all the pointers.
-  void *finished_tag() { return reinterpret_cast<char *>(this); }
-  void *read_tag() { return reinterpret_cast<char *>(this) + 1; }
-  void *write_tag() { return reinterpret_cast<char *>(this) + 2; }
-  void *halfclose_tag() { return reinterpret_cast<char *>(this) + 3; }
-  void *client_metadata_read_tag() {
-    return reinterpret_cast<char *>(this) + 5;
-  }
-  grpc_call *call() { return call_; }
-  grpc_completion_queue *cq() { return cq_; }
-
-  bool is_client_;
-  const RpcMethod *method_;             // not owned
-  grpc_call *call_;                     // not owned
-  grpc_completion_queue *cq_;           // not owned
-  google::protobuf::Message *request_;  // first request, not owned
-  google::protobuf::Message *result_;   // last response, not owned
-
-  bool peer_halfclosed_;
-  bool self_halfclosed_;
-  Status final_status_;
-};
-
-}  // namespace grpc
-
-#endif  // __GRPCPP_INTERNAL_STREAM_STREAM_CONTEXT_H__

test/core/end2end/tests/cancel_after_accept.c

@@ -166,7 +166,8 @@ static void test_cancel_after_accept(grpc_end2end_test_config config,
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(2)));
+                                                      f.server_cq,
+                                                      tag(2)));
   cq_expect_completion(v_server, tag(2), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/early_server_shutdown_finishes_tags.c

@@ -79,7 +79,7 @@ static void drain_cq(grpc_completion_queue *cq) {
 
 static void shutdown_server(grpc_end2end_test_fixture *f) {
   if (!f->server) return;
-  grpc_server_shutdown(f->server);
+  /* don't shutdown, just destroy, to tickle this code edge */
   grpc_server_destroy(f->server);
   f->server = NULL;
 }

test/core/end2end/tests/request_response_with_binary_metadata_and_payload.c

@@ -175,7 +175,8 @@ static void test_request_response_with_metadata_and_payload(
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq, 
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/request_response_with_metadata_and_payload.c

@@ -168,7 +168,8 @@ static void test_request_response_with_metadata_and_payload(
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/request_response_with_payload.c

@@ -162,7 +162,8 @@ static void request_response_with_payload(grpc_end2end_test_fixture f) {
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/request_response_with_trailing_metadata_and_payload.c

@@ -169,7 +169,8 @@ static void test_request_response_with_metadata_and_payload(
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/request_with_large_metadata.c

@@ -166,7 +166,8 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) {
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/request_with_payload.c

@@ -157,7 +157,8 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) {
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/simple_delayed_request.c

@@ -144,7 +144,8 @@ static void simple_delayed_request_body(grpc_end2end_test_config config,
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f->server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f->server_cq, tag(101)));
+                                                      f->server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/end2end/tests/simple_request.c

@@ -150,7 +150,8 @@ static void simple_request_body(grpc_end2end_test_fixture f) {
   GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, &s,
                                                       &call_details,
                                                       &request_metadata_recv,
-                                                      f.server_cq, tag(101)));
+                                                      f.server_cq,
+                                                      tag(101)));
   cq_expect_completion(v_server, tag(101), GRPC_OP_OK);
   cq_verify(v_server);
 

test/core/iomgr/tcp_server_posix_test.c

@@ -66,7 +66,7 @@ static void test_no_op(void) {
 static void test_no_op_with_start(void) {
   grpc_tcp_server *s = grpc_tcp_server_create();
   LOG_TEST();
-  grpc_tcp_server_start(s, NULL, on_connect, NULL);
+  grpc_tcp_server_start(s, NULL, 0, on_connect, NULL);
   grpc_tcp_server_destroy(s);
 }
 
@@ -93,7 +93,7 @@ static void test_no_op_with_port_and_start(void) {
   GPR_ASSERT(
       grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)));
 
-  grpc_tcp_server_start(s, NULL, on_connect, NULL);
+  grpc_tcp_server_start(s, NULL, 0, on_connect, NULL);
 
   grpc_tcp_server_destroy(s);
 }
@@ -120,7 +120,7 @@ static void test_connect(int n) {
   GPR_ASSERT(getsockname(svrfd, (struct sockaddr *)&addr, &addr_len) == 0);
   GPR_ASSERT(addr_len <= sizeof(addr));
 
-  grpc_tcp_server_start(s, NULL, on_connect, NULL);
+  grpc_tcp_server_start(s, NULL, 0, on_connect, NULL);
 
   for (i = 0; i < n; i++) {
     deadline = gpr_time_add(gpr_now(), gpr_time_from_micros(10000000));

test/core/statistics/census_log_tests.c

@@ -35,7 +35,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include "src/core/support/cpu.h"
+#include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>

test/cpp/end2end/async_end2end_test.cc

@@ -0,0 +1,528 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <chrono>
+#include <memory>
+
+#include "test/core/util/test_config.h"
+#include "test/cpp/util/echo_duplicate.pb.h"
+#include "test/cpp/util/echo.pb.h"
+#include "src/cpp/util/time.h"
+#include <grpc++/channel_arguments.h>
+#include <grpc++/channel_interface.h>
+#include <grpc++/client_context.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/credentials.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
+#include <grpc++/status.h>
+#include <grpc++/stream.h>
+#include "test/core/util/port.h"
+#include <gtest/gtest.h>
+
+#include <grpc/grpc.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/time.h>
+
+using grpc::cpp::test::util::EchoRequest;
+using grpc::cpp::test::util::EchoResponse;
+using std::chrono::system_clock;
+
+namespace grpc {
+namespace testing {
+
+namespace {
+
+void* tag(int i) {
+  return (void*)(gpr_intptr)i;
+}
+
+void verify_ok(CompletionQueue* cq, int i, bool expect_ok) {
+  bool ok;
+  void* got_tag;
+  EXPECT_TRUE(cq->Next(&got_tag, &ok));
+  EXPECT_EQ(expect_ok, ok);
+  EXPECT_EQ(tag(i), got_tag);
+}
+
+class AsyncEnd2endTest : public ::testing::Test {
+ protected:
+  AsyncEnd2endTest() : service_(&srv_cq_) {}
+
+  void SetUp() override {
+    int port = grpc_pick_unused_port_or_die();
+    server_address_ << "localhost:" << port;
+    // Setup server
+    ServerBuilder builder;
+    builder.AddPort(server_address_.str());
+    builder.RegisterAsyncService(&service_);
+    server_ = builder.BuildAndStart();
+  }
+
+  void TearDown() override { server_->Shutdown(); }
+
+  void ResetStub() {
+    std::shared_ptr<ChannelInterface> channel =
+        CreateChannel(server_address_.str(), ChannelArguments());
+    stub_.reset(grpc::cpp::test::util::TestService::NewStub(channel));
+  }
+
+  void server_ok(int i) {
+    verify_ok(&srv_cq_, i, true);
+  }
+  void client_ok(int i) {
+    verify_ok(&cli_cq_, i , true);
+  }
+  void server_fail(int i) {
+    verify_ok(&srv_cq_, i, false);
+  }
+  void client_fail(int i) {
+    verify_ok(&cli_cq_, i, false);
+  }
+
+  void SendRpc(int num_rpcs) {
+    for (int i = 0; i < num_rpcs; i++) {
+      EchoRequest send_request;
+      EchoRequest recv_request;
+      EchoResponse send_response;
+      EchoResponse recv_response;
+      Status recv_status;
+
+      ClientContext cli_ctx;
+      ServerContext srv_ctx;
+      grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+      send_request.set_message("Hello");
+      stub_->Echo(
+          &cli_ctx, send_request, &recv_response, &recv_status, &cli_cq_, tag(1));
+
+      service_.RequestEcho(
+          &srv_ctx, &recv_request, &response_writer, &srv_cq_, tag(2));
+
+      server_ok(2);
+      EXPECT_EQ(send_request.message(), recv_request.message());
+
+      send_response.set_message(recv_request.message());
+      response_writer.Finish(send_response, Status::OK, tag(3));
+
+      server_ok(3);
+
+      client_ok(1);
+
+      EXPECT_EQ(send_response.message(), recv_response.message());
+      EXPECT_TRUE(recv_status.IsOk());
+    }
+  }
+
+  CompletionQueue cli_cq_;
+  CompletionQueue srv_cq_;
+  std::unique_ptr<grpc::cpp::test::util::TestService::Stub> stub_;
+  std::unique_ptr<Server> server_;
+  grpc::cpp::test::util::TestService::AsyncService service_;
+  std::ostringstream server_address_;
+};
+
+TEST_F(AsyncEnd2endTest, SimpleRpc) {
+  ResetStub();
+  SendRpc(1);
+}
+
+TEST_F(AsyncEnd2endTest, SequentialRpcs) {
+  ResetStub();
+  SendRpc(10);
+}
+
+// Two pings and a final pong.
+TEST_F(AsyncEnd2endTest, SimpleClientStreaming) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  ServerAsyncReader<EchoResponse, EchoRequest> srv_stream(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::unique_ptr<ClientAsyncWriter<EchoRequest> > cli_stream(
+      stub_->RequestStream(&cli_ctx, &recv_response, &cli_cq_, tag(1)));
+
+  service_.RequestRequestStream(
+      &srv_ctx, &srv_stream, &srv_cq_, tag(2));
+
+  server_ok(2);
+  client_ok(1);
+
+  cli_stream->Write(send_request, tag(3));
+  client_ok(3);
+
+  srv_stream.Read(&recv_request, tag(4));
+  server_ok(4);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+
+  cli_stream->Write(send_request, tag(5));
+  client_ok(5);
+
+  srv_stream.Read(&recv_request, tag(6));
+  server_ok(6);
+
+  EXPECT_EQ(send_request.message(), recv_request.message());
+  cli_stream->WritesDone(tag(7));
+  client_ok(7);
+
+  srv_stream.Read(&recv_request, tag(8));
+  server_fail(8);
+
+  send_response.set_message(recv_request.message());
+  srv_stream.Finish(send_response, Status::OK, tag(9));
+  server_ok(9);
+
+  cli_stream->Finish(&recv_status, tag(10));
+  client_ok(10);
+
+  EXPECT_EQ(send_response.message(), recv_response.message());
+  EXPECT_TRUE(recv_status.IsOk());
+}
+
+// One ping, two pongs.
+TEST_F(AsyncEnd2endTest, SimpleServerStreaming) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  ServerAsyncWriter<EchoResponse> srv_stream(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::unique_ptr<ClientAsyncReader<EchoResponse> > cli_stream(
+      stub_->ResponseStream(&cli_ctx, send_request, &cli_cq_, tag(1)));
+
+  service_.RequestResponseStream(
+      &srv_ctx, &recv_request, &srv_stream, &srv_cq_, tag(2));
+
+  server_ok(2);
+  client_ok(1);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+
+  send_response.set_message(recv_request.message());
+  srv_stream.Write(send_response, tag(3));
+  server_ok(3);
+
+  cli_stream->Read(&recv_response, tag(4));
+  client_ok(4);
+  EXPECT_EQ(send_response.message(), recv_response.message());
+
+  srv_stream.Write(send_response, tag(5));
+  server_ok(5);
+
+  cli_stream->Read(&recv_response, tag(6));
+  client_ok(6);
+  EXPECT_EQ(send_response.message(), recv_response.message());
+
+  srv_stream.Finish(Status::OK, tag(7));
+  server_ok(7);
+
+  cli_stream->Read(&recv_response, tag(8));
+  client_fail(8);
+
+  cli_stream->Finish(&recv_status, tag(9));
+  client_ok(9);
+
+  EXPECT_TRUE(recv_status.IsOk());
+}
+
+// One ping, one pong.
+TEST_F(AsyncEnd2endTest, SimpleBidiStreaming) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  ServerAsyncReaderWriter<EchoResponse, EchoRequest> srv_stream(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::unique_ptr<ClientAsyncReaderWriter<EchoRequest, EchoResponse> >
+      cli_stream(stub_->BidiStream(&cli_ctx, &cli_cq_, tag(1)));
+
+  service_.RequestBidiStream(
+      &srv_ctx, &srv_stream, &srv_cq_, tag(2));
+
+  server_ok(2);
+  client_ok(1);
+
+  cli_stream->Write(send_request, tag(3));
+  client_ok(3);
+
+  srv_stream.Read(&recv_request, tag(4));
+  server_ok(4);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+
+  send_response.set_message(recv_request.message());
+  srv_stream.Write(send_response, tag(5));
+  server_ok(5);
+
+  cli_stream->Read(&recv_response, tag(6));
+  client_ok(6);
+  EXPECT_EQ(send_response.message(), recv_response.message());
+
+  cli_stream->WritesDone(tag(7));
+  client_ok(7);
+
+  srv_stream.Read(&recv_request, tag(8));
+  server_fail(8);
+
+  srv_stream.Finish(Status::OK, tag(9));
+  server_ok(9);
+
+  cli_stream->Finish(&recv_status, tag(10));
+  client_ok(10);
+
+  EXPECT_TRUE(recv_status.IsOk());
+}
+
+// Metadata tests
+TEST_F(AsyncEnd2endTest, ClientInitialMetadataRpc) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::pair<grpc::string, grpc::string> meta1("key1", "val1");
+  std::pair<grpc::string, grpc::string> meta2("key2", "val2");
+  cli_ctx.AddMetadata(meta1.first, meta1.second);
+  cli_ctx.AddMetadata(meta2.first, meta2.second);
+
+  stub_->Echo(
+      &cli_ctx, send_request, &recv_response, &recv_status, &cli_cq_, tag(1));
+
+  service_.RequestEcho(
+      &srv_ctx, &recv_request, &response_writer, &srv_cq_, tag(2));
+  server_ok(2);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+  auto client_initial_metadata = srv_ctx.client_metadata();
+  EXPECT_EQ(meta1.second, client_initial_metadata.find(meta1.first)->second);
+  EXPECT_EQ(meta2.second, client_initial_metadata.find(meta2.first)->second);
+  EXPECT_EQ(2, client_initial_metadata.size());
+
+  send_response.set_message(recv_request.message());
+  response_writer.Finish(send_response, Status::OK, tag(3));
+
+  server_ok(3);
+
+  client_ok(1);
+
+  EXPECT_EQ(send_response.message(), recv_response.message());
+  EXPECT_TRUE(recv_status.IsOk());
+}
+
+TEST_F(AsyncEnd2endTest, ServerInitialMetadataRpc) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::pair<grpc::string, grpc::string> meta1("key1", "val1");
+  std::pair<grpc::string, grpc::string> meta2("key2", "val2");
+
+  stub_->Echo(
+      &cli_ctx, send_request, &recv_response, &recv_status, &cli_cq_, tag(1));
+
+  service_.RequestEcho(
+      &srv_ctx, &recv_request, &response_writer, &srv_cq_, tag(2));
+  server_ok(2);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+  srv_ctx.AddInitialMetadata(meta1.first, meta1.second);
+  srv_ctx.AddInitialMetadata(meta2.first, meta2.second);
+  response_writer.SendInitialMetadata(tag(3));
+  server_ok(3);
+
+  send_response.set_message(recv_request.message());
+  response_writer.Finish(send_response, Status::OK, tag(4));
+
+  server_ok(4);
+
+  client_ok(1);
+
+  EXPECT_EQ(send_response.message(), recv_response.message());
+  EXPECT_TRUE(recv_status.IsOk());
+  auto server_initial_metadata = cli_ctx.GetServerInitialMetadata();
+  EXPECT_EQ(meta1.second, server_initial_metadata.find(meta1.first)->second);
+  EXPECT_EQ(meta2.second, server_initial_metadata.find(meta2.first)->second);
+  EXPECT_EQ(2, server_initial_metadata.size());
+}
+
+TEST_F(AsyncEnd2endTest, ServerTrailingMetadataRpc) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::pair<grpc::string, grpc::string> meta1("key1", "val1");
+  std::pair<grpc::string, grpc::string> meta2("key2", "val2");
+
+  stub_->Echo(
+      &cli_ctx, send_request, &recv_response, &recv_status, &cli_cq_, tag(1));
+
+  service_.RequestEcho(
+      &srv_ctx, &recv_request, &response_writer, &srv_cq_, tag(2));
+  server_ok(2);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+  response_writer.SendInitialMetadata(tag(3));
+  server_ok(3);
+
+  send_response.set_message(recv_request.message());
+  srv_ctx.AddTrailingMetadata(meta1.first, meta1.second);
+  srv_ctx.AddTrailingMetadata(meta2.first, meta2.second);
+  response_writer.Finish(send_response, Status::OK, tag(4));
+
+  server_ok(4);
+
+  client_ok(1);
+
+  EXPECT_EQ(send_response.message(), recv_response.message());
+  EXPECT_TRUE(recv_status.IsOk());
+  auto server_trailing_metadata = cli_ctx.GetServerTrailingMetadata();
+  EXPECT_EQ(meta1.second, server_trailing_metadata.find(meta1.first)->second);
+  EXPECT_EQ(meta2.second, server_trailing_metadata.find(meta2.first)->second);
+  EXPECT_EQ(2, server_trailing_metadata.size());
+}
+
+TEST_F(AsyncEnd2endTest, MetadataRpc) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::pair<grpc::string, grpc::string> meta1("key1", "val1");
+  std::pair<grpc::string, grpc::string> meta2("key2-bin", {"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc", 13});
+  std::pair<grpc::string, grpc::string> meta3("key3", "val3");
+  std::pair<grpc::string, grpc::string> meta6("key4-bin", {"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d", 14});
+  std::pair<grpc::string, grpc::string> meta5("key5", "val5");
+  std::pair<grpc::string, grpc::string> meta4("key6-bin", {"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee", 15});
+
+  cli_ctx.AddMetadata(meta1.first, meta1.second);
+  cli_ctx.AddMetadata(meta2.first, meta2.second);
+
+  stub_->Echo(
+      &cli_ctx, send_request, &recv_response, &recv_status, &cli_cq_, tag(1));
+
+  service_.RequestEcho(
+      &srv_ctx, &recv_request, &response_writer, &srv_cq_, tag(2));
+  server_ok(2);
+  EXPECT_EQ(send_request.message(), recv_request.message());
+  auto client_initial_metadata = srv_ctx.client_metadata();
+  EXPECT_EQ(meta1.second, client_initial_metadata.find(meta1.first)->second);
+  EXPECT_EQ(meta2.second, client_initial_metadata.find(meta2.first)->second);
+  EXPECT_EQ(2, client_initial_metadata.size());
+
+  srv_ctx.AddInitialMetadata(meta3.first, meta3.second);
+  srv_ctx.AddInitialMetadata(meta4.first, meta4.second);
+  response_writer.SendInitialMetadata(tag(3));
+  server_ok(3);
+
+  send_response.set_message(recv_request.message());
+  srv_ctx.AddTrailingMetadata(meta5.first, meta5.second);
+  srv_ctx.AddTrailingMetadata(meta6.first, meta6.second);
+  response_writer.Finish(send_response, Status::OK, tag(4));
+
+  server_ok(4);
+
+  client_ok(1);
+
+  EXPECT_EQ(send_response.message(), recv_response.message());
+  EXPECT_TRUE(recv_status.IsOk());
+  auto server_initial_metadata = cli_ctx.GetServerInitialMetadata();
+  EXPECT_EQ(meta3.second, server_initial_metadata.find(meta3.first)->second);
+  EXPECT_EQ(meta4.second, server_initial_metadata.find(meta4.first)->second);
+  EXPECT_EQ(2, server_initial_metadata.size());
+  auto server_trailing_metadata = cli_ctx.GetServerTrailingMetadata();
+  EXPECT_EQ(meta5.second, server_trailing_metadata.find(meta5.first)->second);
+  EXPECT_EQ(meta6.second, server_trailing_metadata.find(meta6.first)->second);
+  EXPECT_EQ(2, server_trailing_metadata.size());
+}
+}  // namespace
+}  // namespace testing
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  grpc_test_init(argc, argv);
+  grpc_init();
+  ::testing::InitGoogleTest(&argc, argv);
+  int result = RUN_ALL_TESTS();
+  grpc_shutdown();
+  google::protobuf::ShutdownProtobufLibrary();
+  return result;
+}

test/cpp/end2end/async_test_server.cc

@@ -1,154 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "test/cpp/end2end/async_test_server.h"
-
-#include <chrono>
-
-#include <grpc/support/log.h>
-#include "src/cpp/proto/proto_utils.h"
-#include "test/cpp/util/echo.pb.h"
-#include <grpc++/async_server.h>
-#include <grpc++/async_server_context.h>
-#include <grpc++/completion_queue.h>
-#include <grpc++/status.h>
-#include <gtest/gtest.h>
-
-using grpc::cpp::test::util::EchoRequest;
-using grpc::cpp::test::util::EchoResponse;
-
-using std::chrono::duration_cast;
-using std::chrono::microseconds;
-using std::chrono::seconds;
-using std::chrono::system_clock;
-
-namespace grpc {
-namespace testing {
-
-AsyncTestServer::AsyncTestServer() : server_(&cq_), cq_drained_(false) {}
-
-AsyncTestServer::~AsyncTestServer() {}
-
-void AsyncTestServer::AddPort(const grpc::string& addr) {
-  server_.AddPort(addr);
-}
-
-void AsyncTestServer::Start() { server_.Start(); }
-
-// Return true if deadline actual is within 0.5s from expected.
-bool DeadlineMatched(const system_clock::time_point& actual,
-                     const system_clock::time_point& expected) {
-  microseconds diff_usecs = duration_cast<microseconds>(expected - actual);
-  gpr_log(GPR_INFO, "diff_usecs= %d", diff_usecs.count());
-  return diff_usecs.count() < 500000 && diff_usecs.count() > -500000;
-}
-
-void AsyncTestServer::RequestOneRpc() { server_.RequestOneRpc(); }
-
-void AsyncTestServer::MainLoop() {
-  EchoRequest request;
-  EchoResponse response;
-  void* tag = nullptr;
-
-  RequestOneRpc();
-
-  while (true) {
-    CompletionQueue::CompletionType t = cq_.Next(&tag);
-    AsyncServerContext* server_context = static_cast<AsyncServerContext*>(tag);
-    switch (t) {
-      case CompletionQueue::SERVER_RPC_NEW:
-        gpr_log(GPR_INFO, "SERVER_RPC_NEW %p", server_context);
-        if (server_context) {
-          EXPECT_EQ(server_context->method(), "/foo");
-          // TODO(ctiller): verify deadline
-          server_context->Accept(cq_.cq());
-          // Handle only one rpc at a time.
-          RequestOneRpc();
-          server_context->StartRead(&request);
-        }
-        break;
-      case CompletionQueue::RPC_END:
-        gpr_log(GPR_INFO, "RPC_END %p", server_context);
-        delete server_context;
-        break;
-      case CompletionQueue::SERVER_READ_OK:
-        gpr_log(GPR_INFO, "SERVER_READ_OK %p", server_context);
-        response.set_message(request.message());
-        server_context->StartWrite(response, 0);
-        break;
-      case CompletionQueue::SERVER_READ_ERROR:
-        gpr_log(GPR_INFO, "SERVER_READ_ERROR %p", server_context);
-        server_context->StartWriteStatus(Status::OK);
-        break;
-      case CompletionQueue::HALFCLOSE_OK:
-        gpr_log(GPR_INFO, "HALFCLOSE_OK %p", server_context);
-        // Do nothing, just wait for RPC_END.
-        break;
-      case CompletionQueue::SERVER_WRITE_OK:
-        gpr_log(GPR_INFO, "SERVER_WRITE_OK %p", server_context);
-        server_context->StartRead(&request);
-        break;
-      case CompletionQueue::SERVER_WRITE_ERROR:
-        EXPECT_TRUE(0);
-        break;
-      case CompletionQueue::QUEUE_CLOSED: {
-        gpr_log(GPR_INFO, "QUEUE_CLOSED");
-        HandleQueueClosed();
-        return;
-      }
-      default:
-        EXPECT_TRUE(0);
-        break;
-    }
-  }
-}
-
-void AsyncTestServer::HandleQueueClosed() {
-  std::unique_lock<std::mutex> lock(cq_drained_mu_);
-  cq_drained_ = true;
-  cq_drained_cv_.notify_all();
-}
-
-void AsyncTestServer::Shutdown() {
-  // The server need to be shut down before cq_ as grpc_server flushes all
-  // pending requested calls to the completion queue at shutdown.
-  server_.Shutdown();
-  cq_.Shutdown();
-  std::unique_lock<std::mutex> lock(cq_drained_mu_);
-  while (!cq_drained_) {
-    cq_drained_cv_.wait(lock);
-  }
-}
-
-}  // namespace testing
-}  // namespace grpc

test/cpp/end2end/async_test_server.h

@@ -1,75 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef __GRPCPP_TEST_END2END_ASYNC_TEST_SERVER_H__
-#define __GRPCPP_TEST_END2END_ASYNC_TEST_SERVER_H__
-
-#include <condition_variable>
-#include <mutex>
-#include <string>
-
-#include <grpc++/async_server.h>
-#include <grpc++/completion_queue.h>
-
-namespace grpc {
-
-namespace testing {
-
-class AsyncTestServer {
- public:
-  AsyncTestServer();
-  virtual ~AsyncTestServer();
-
-  void AddPort(const grpc::string& addr);
-  void Start();
-  void RequestOneRpc();
-  virtual void MainLoop();
-  void Shutdown();
-
-  CompletionQueue* completion_queue() { return &cq_; }
-
- protected:
-  void HandleQueueClosed();
-
- private:
-  CompletionQueue cq_;
-  AsyncServer server_;
-  bool cq_drained_;
-  std::mutex cq_drained_mu_;
-  std::condition_variable cq_drained_cv_;
-};
-
-}  // namespace testing
-}  // namespace grpc
-
-#endif  // __GRPCPP_TEST_END2END_ASYNC_TEST_SERVER_H__

test/cpp/end2end/end2end_test.cc

@@ -38,6 +38,7 @@
 #include "test/cpp/util/echo_duplicate.pb.h"
 #include "test/cpp/util/echo.pb.h"
 #include "src/cpp/util/time.h"
+#include "src/cpp/server/thread_pool.h"
 #include <grpc++/channel_arguments.h>
 #include <grpc++/channel_interface.h>
 #include <grpc++/client_context.h>
@@ -76,6 +77,7 @@ void MaybeEchoDeadline(ServerContext* context, const EchoRequest* request,
     response->mutable_param()->set_request_deadline(deadline.tv_sec);
   }
 }
+
 }  // namespace
 
 class TestServiceImpl : public ::grpc::cpp::test::util::TestService::Service {
@@ -141,14 +143,17 @@ class TestServiceImplDupPkg
 
 class End2endTest : public ::testing::Test {
  protected:
+  End2endTest() : thread_pool_(2) {}
+
   void SetUp() override {
     int port = grpc_pick_unused_port_or_die();
     server_address_ << "localhost:" << port;
     // Setup server
     ServerBuilder builder;
     builder.AddPort(server_address_.str());
-    builder.RegisterService(service_.service());
+    builder.RegisterService(&service_);
-    builder.RegisterService(dup_pkg_service_.service());
+    builder.RegisterService(&dup_pkg_service_);
+    builder.SetThreadPool(&thread_pool_);
     server_ = builder.BuildAndStart();
   }
 
@@ -165,6 +170,7 @@ class End2endTest : public ::testing::Test {
   std::ostringstream server_address_;
   TestServiceImpl service_;
   TestServiceImplDupPkg dup_pkg_service_;
+  ThreadPool thread_pool_;
 };
 
 static void SendRpc(grpc::cpp::test::util::TestService::Stub* stub,
@@ -290,7 +296,7 @@ TEST_F(End2endTest, RequestStreamOneRequest) {
   request.set_message("hello");
   EXPECT_TRUE(stream->Write(request));
   stream->WritesDone();
-  Status s = stream->Wait();
+  Status s = stream->Finish();
   EXPECT_EQ(response.message(), request.message());
   EXPECT_TRUE(s.IsOk());
 
@@ -308,7 +314,7 @@ TEST_F(End2endTest, RequestStreamTwoRequests) {
   EXPECT_TRUE(stream->Write(request));
   EXPECT_TRUE(stream->Write(request));
   stream->WritesDone();
-  Status s = stream->Wait();
+  Status s = stream->Finish();
   EXPECT_EQ(response.message(), "hellohello");
   EXPECT_TRUE(s.IsOk());
 
@@ -323,7 +329,7 @@ TEST_F(End2endTest, ResponseStream) {
   request.set_message("hello");
 
   ClientReader<EchoResponse>* stream =
-      stub_->ResponseStream(&context, &request);
+      stub_->ResponseStream(&context, request);
   EXPECT_TRUE(stream->Read(&response));
   EXPECT_EQ(response.message(), request.message() + "0");
   EXPECT_TRUE(stream->Read(&response));
@@ -332,7 +338,7 @@ TEST_F(End2endTest, ResponseStream) {
   EXPECT_EQ(response.message(), request.message() + "2");
   EXPECT_FALSE(stream->Read(&response));
 
-  Status s = stream->Wait();
+  Status s = stream->Finish();
   EXPECT_TRUE(s.IsOk());
 
   delete stream;
@@ -366,7 +372,7 @@ TEST_F(End2endTest, BidiStream) {
   stream->WritesDone();
   EXPECT_FALSE(stream->Read(&response));
 
-  Status s = stream->Wait();
+  Status s = stream->Finish();
   EXPECT_TRUE(s.IsOk());
 
   delete stream;
@@ -422,7 +428,7 @@ TEST_F(End2endTest, BadCredentials) {
   ClientContext context2;
   ClientReaderWriter<EchoRequest, EchoResponse>* stream =
       stub->BidiStream(&context2);
-  s = stream->Wait();
+  s = stream->Finish();
   EXPECT_FALSE(s.IsOk());
   EXPECT_EQ(StatusCode::UNKNOWN, s.code());
   EXPECT_EQ("Rpc sent on a lame channel.", s.details());
@@ -439,5 +445,6 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   int result = RUN_ALL_TESTS();
   grpc_shutdown();
+  google::protobuf::ShutdownProtobufLibrary();
   return result;
 }

test/cpp/end2end/sync_client_async_server_test.cc

@@ -1,236 +0,0 @@
-/*
- *
- * Copyright 2014, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <chrono>
-#include <memory>
-#include <sstream>
-#include <string>
-
-#include <grpc/grpc.h>
-#include <grpc/support/thd.h>
-#include "test/cpp/util/echo.pb.h"
-#include <grpc++/channel_arguments.h>
-#include <grpc++/channel_interface.h>
-#include <grpc++/client_context.h>
-#include <grpc++/create_channel.h>
-#include <grpc++/impl/internal_stub.h>
-#include <grpc++/impl/rpc_method.h>
-#include <grpc++/status.h>
-#include <grpc++/stream.h>
-#include "test/cpp/end2end/async_test_server.h"
-#include "test/core/util/port.h"
-#include <gtest/gtest.h>
-
-using grpc::cpp::test::util::EchoRequest;
-using grpc::cpp::test::util::EchoResponse;
-
-using std::chrono::duration_cast;
-using std::chrono::microseconds;
-using std::chrono::seconds;
-using std::chrono::system_clock;
-
-using grpc::testing::AsyncTestServer;
-
-namespace grpc {
-namespace {
-
-void ServerLoop(void* s) {
-  AsyncTestServer* server = static_cast<AsyncTestServer*>(s);
-  server->MainLoop();
-}
-
-class End2endTest : public ::testing::Test {
- protected:
-  void SetUp() override {
-    int port = grpc_pick_unused_port_or_die();
-    // TODO(yangg) protobuf has a StringPrintf, maybe use that
-    std::ostringstream oss;
-    oss << "[::]:" << port;
-    // Setup server
-    server_.reset(new AsyncTestServer());
-    server_->AddPort(oss.str());
-    server_->Start();
-
-    RunServerThread();
-
-    // Setup client
-    oss.str("");
-    oss << "127.0.0.1:" << port;
-    std::shared_ptr<ChannelInterface> channel =
-        CreateChannel(oss.str(), ChannelArguments());
-    stub_.set_channel(channel);
-  }
-
-  void RunServerThread() {
-    gpr_thd_id id;
-    EXPECT_TRUE(gpr_thd_new(&id, ServerLoop, server_.get(), NULL));
-  }
-
-  void TearDown() override { server_->Shutdown(); }
-
-  std::unique_ptr<AsyncTestServer> server_;
-  InternalStub stub_;
-};
-
-TEST_F(End2endTest, NoOpTest) { EXPECT_TRUE(stub_.channel() != nullptr); }
-
-TEST_F(End2endTest, SimpleRpc) {
-  EchoRequest request;
-  request.set_message("hello");
-  EchoResponse result;
-  ClientContext context;
-  RpcMethod method("/foo");
-  std::chrono::system_clock::time_point deadline =
-      std::chrono::system_clock::now() + std::chrono::seconds(10);
-  context.set_absolute_deadline(deadline);
-  Status s =
-      stub_.channel()->StartBlockingRpc(method, &context, request, &result);
-  EXPECT_EQ(result.message(), request.message());
-  EXPECT_TRUE(s.IsOk());
-}
-
-TEST_F(End2endTest, KSequentialSimpleRpcs) {
-  int k = 3;
-  for (int i = 0; i < k; i++) {
-    EchoRequest request;
-    request.set_message("hello");
-    EchoResponse result;
-    ClientContext context;
-    RpcMethod method("/foo");
-    std::chrono::system_clock::time_point deadline =
-        std::chrono::system_clock::now() + std::chrono::seconds(10);
-    context.set_absolute_deadline(deadline);
-    Status s =
-        stub_.channel()->StartBlockingRpc(method, &context, request, &result);
-    EXPECT_EQ(result.message(), request.message());
-    EXPECT_TRUE(s.IsOk());
-  }
-}
-
-TEST_F(End2endTest, OnePingpongBidiStream) {
-  EchoRequest request;
-  request.set_message("hello");
-  EchoResponse result;
-  ClientContext context;
-  RpcMethod method("/foo", RpcMethod::RpcType::BIDI_STREAMING);
-  std::chrono::system_clock::time_point deadline =
-      std::chrono::system_clock::now() + std::chrono::seconds(10);
-  context.set_absolute_deadline(deadline);
-  StreamContextInterface* stream_interface =
-      stub_.channel()->CreateStream(method, &context, nullptr, nullptr);
-  std::unique_ptr<ClientReaderWriter<EchoRequest, EchoResponse>> stream(
-      new ClientReaderWriter<EchoRequest, EchoResponse>(stream_interface));
-  EXPECT_TRUE(stream->Write(request));
-  EXPECT_TRUE(stream->Read(&result));
-  stream->WritesDone();
-  EXPECT_FALSE(stream->Read(&result));
-  Status s = stream->Wait();
-  EXPECT_EQ(result.message(), request.message());
-  EXPECT_TRUE(s.IsOk());
-}
-
-TEST_F(End2endTest, TwoPingpongBidiStream) {
-  EchoRequest request;
-  request.set_message("hello");
-  EchoResponse result;
-  ClientContext context;
-  RpcMethod method("/foo", RpcMethod::RpcType::BIDI_STREAMING);
-  std::chrono::system_clock::time_point deadline =
-      std::chrono::system_clock::now() + std::chrono::seconds(10);
-  context.set_absolute_deadline(deadline);
-  StreamContextInterface* stream_interface =
-      stub_.channel()->CreateStream(method, &context, nullptr, nullptr);
-  std::unique_ptr<ClientReaderWriter<EchoRequest, EchoResponse>> stream(
-      new ClientReaderWriter<EchoRequest, EchoResponse>(stream_interface));
-  EXPECT_TRUE(stream->Write(request));
-  EXPECT_TRUE(stream->Read(&result));
-  EXPECT_EQ(result.message(), request.message());
-  EXPECT_TRUE(stream->Write(request));
-  EXPECT_TRUE(stream->Read(&result));
-  EXPECT_EQ(result.message(), request.message());
-  stream->WritesDone();
-  EXPECT_FALSE(stream->Read(&result));
-  Status s = stream->Wait();
-  EXPECT_TRUE(s.IsOk());
-}
-
-TEST_F(End2endTest, OnePingpongClientStream) {
-  EchoRequest request;
-  request.set_message("hello");
-  EchoResponse result;
-  ClientContext context;
-  RpcMethod method("/foo", RpcMethod::RpcType::CLIENT_STREAMING);
-  std::chrono::system_clock::time_point deadline =
-      std::chrono::system_clock::now() + std::chrono::seconds(10);
-  context.set_absolute_deadline(deadline);
-  StreamContextInterface* stream_interface =
-      stub_.channel()->CreateStream(method, &context, nullptr, &result);
-  std::unique_ptr<ClientWriter<EchoRequest>> stream(
-      new ClientWriter<EchoRequest>(stream_interface));
-  EXPECT_TRUE(stream->Write(request));
-  stream->WritesDone();
-  Status s = stream->Wait();
-  EXPECT_EQ(result.message(), request.message());
-  EXPECT_TRUE(s.IsOk());
-}
-
-TEST_F(End2endTest, OnePingpongServerStream) {
-  EchoRequest request;
-  request.set_message("hello");
-  EchoResponse result;
-  ClientContext context;
-  RpcMethod method("/foo", RpcMethod::RpcType::SERVER_STREAMING);
-  std::chrono::system_clock::time_point deadline =
-      std::chrono::system_clock::now() + std::chrono::seconds(10);
-  context.set_absolute_deadline(deadline);
-  StreamContextInterface* stream_interface =
-      stub_.channel()->CreateStream(method, &context, &request, nullptr);
-  std::unique_ptr<ClientReader<EchoResponse>> stream(
-      new ClientReader<EchoResponse>(stream_interface));
-  EXPECT_TRUE(stream->Read(&result));
-  EXPECT_FALSE(stream->Read(nullptr));
-  Status s = stream->Wait();
-  EXPECT_EQ(result.message(), request.message());
-  EXPECT_TRUE(s.IsOk());
-}
-
-}  // namespace
-}  // namespace grpc
-
-int main(int argc, char** argv) {
-  grpc_init();
-  ::testing::InitGoogleTest(&argc, argv);
-  int result = RUN_ALL_TESTS();
-  grpc_shutdown();
-  return result;
-}

test/cpp/interop/client.cc

@@ -248,7 +248,7 @@ void DoRequestStreaming() {
     aggregated_payload_size += request_stream_sizes[i];
   }
   stream->WritesDone();
-  grpc::Status s = stream->Wait();
+  grpc::Status s = stream->Finish();
 
   GPR_ASSERT(response.aggregated_payload_size() == aggregated_payload_size);
   GPR_ASSERT(s.IsOk());
@@ -269,7 +269,7 @@ void DoResponseStreaming() {
   }
   StreamingOutputCallResponse response;
   std::unique_ptr<grpc::ClientReader<StreamingOutputCallResponse>> stream(
-      stub->StreamingOutputCall(&context, &request));
+      stub->StreamingOutputCall(&context, request));
 
   unsigned int i = 0;
   while (stream->Read(&response)) {
@@ -278,7 +278,7 @@ void DoResponseStreaming() {
     ++i;
   }
   GPR_ASSERT(response_stream_sizes.size() == i);
-  grpc::Status s = stream->Wait();
+  grpc::Status s = stream->Finish();
 
   GPR_ASSERT(s.IsOk());
   gpr_log(GPR_INFO, "Response streaming done.");
@@ -299,7 +299,7 @@ void DoResponseStreamingWithSlowConsumer() {
   }
   StreamingOutputCallResponse response;
   std::unique_ptr<grpc::ClientReader<StreamingOutputCallResponse>> stream(
-      stub->StreamingOutputCall(&context, &request));
+      stub->StreamingOutputCall(&context, request));
 
   int i = 0;
   while (stream->Read(&response)) {
@@ -311,7 +311,7 @@ void DoResponseStreamingWithSlowConsumer() {
     ++i;
   }
   GPR_ASSERT(kNumResponseMessages == i);
-  grpc::Status s = stream->Wait();
+  grpc::Status s = stream->Finish();
 
   GPR_ASSERT(s.IsOk());
   gpr_log(GPR_INFO, "Response streaming done.");
@@ -345,7 +345,7 @@ void DoHalfDuplex() {
     ++i;
   }
   GPR_ASSERT(response_stream_sizes.size() == i);
-  grpc::Status s = stream->Wait();
+  grpc::Status s = stream->Finish();
   GPR_ASSERT(s.IsOk());
   gpr_log(GPR_INFO, "Half-duplex streaming rpc done.");
 }
@@ -378,7 +378,7 @@ void DoPingPong() {
 
   stream->WritesDone();
   GPR_ASSERT(!stream->Read(&response));
-  grpc::Status s = stream->Wait();
+  grpc::Status s = stream->Finish();
   GPR_ASSERT(s.IsOk());
   gpr_log(GPR_INFO, "Ping pong streaming done.");
 }

test/cpp/interop/server.cc

@@ -200,7 +200,7 @@ void RunServer() {
 
   ServerBuilder builder;
   builder.AddPort(server_address.str());
-  builder.RegisterService(service.service());
+  builder.RegisterService(&service);
   if (FLAGS_enable_ssl) {
     SslServerCredentialsOptions ssl_opts = {
         "", {{test_server1_key, test_server1_cert}}};

test/cpp/qps/server.cc

@@ -128,7 +128,7 @@ static void RunServer() {
 
   ServerBuilder builder;
   builder.AddPort(server_address);
-  builder.RegisterService(service.service());
+  builder.RegisterService(&service);
 
   std::unique_ptr<ThreadPool> pool(new ThreadPool(FLAGS_server_threads));
   builder.SetThreadPool(pool.get());

tools/run_tests/tests.json

@@ -269,6 +269,10 @@
     "language": "c", 
     "name": "transport_metadata_test"
   }, 
+  {
+    "language": "c++", 
+    "name": "async_end2end_test"
+  }, 
   {
     "language": "c++", 
     "name": "channel_arguments_test"
@@ -293,10 +297,6 @@
     "language": "c++", 
     "name": "status_test"
   }, 
-  {
-    "language": "c++", 
-    "name": "sync_client_async_server_test"
-  }, 
   {
     "language": "c++", 
     "name": "thread_pool_test"