Merge branch 'master' of github.com:google/grpc into grpc-win32

build.json

@@ -76,6 +76,7 @@
         "src/core/statistics/census_tracing.h",
         "src/core/statistics/hash_table.h",
         "src/core/statistics/window_stats.h",
+        "src/core/surface/byte_buffer_queue.h",
         "src/core/surface/call.h",
         "src/core/surface/channel.h",
         "src/core/surface/client.h",
@@ -168,8 +169,10 @@
         "src/core/statistics/hash_table.c",
         "src/core/statistics/window_stats.c",
         "src/core/surface/byte_buffer.c",
+        "src/core/surface/byte_buffer_queue.c",
         "src/core/surface/byte_buffer_reader.c",
         "src/core/surface/call.c",
+        "src/core/surface/call_details.c",
         "src/core/surface/channel.c",
         "src/core/surface/channel_create.c",
         "src/core/surface/client.c",
@@ -177,6 +180,7 @@
         "src/core/surface/event_string.c",
         "src/core/surface/init.c",
         "src/core/surface/lame_client.c",
+        "src/core/surface/metadata_array.c",
         "src/core/surface/secure_channel_create.c",
         "src/core/surface/secure_server_create.c",
         "src/core/surface/server.c",
@@ -235,8 +239,11 @@
       ],
       "headers": [
         "src/core/support/cpu.h",
+        "src/core/support/env.h",
+        "src/core/support/file.h",
         "src/core/support/murmur_hash.h",
         "src/core/support/string.h",
+        "src/core/support/string_win32.h",
         "src/core/support/thd_internal.h"
       ],
       "src": [
@@ -245,6 +252,12 @@
         "src/core/support/cmdline.c",
         "src/core/support/cpu_linux.c",
         "src/core/support/cpu_posix.c",
+        "src/core/support/env_linux.c",
+        "src/core/support/env_posix.c",
+        "src/core/support/env_win32.c",
+        "src/core/support/file.c",
+        "src/core/support/file_posix.c",
+        "src/core/support/file_win32.c",
         "src/core/support/histogram.c",
         "src/core/support/host_port.c",
         "src/core/support/log.c",
@@ -324,6 +337,18 @@
       "secure": true,
       "vs_project_guid": "{29D16885-7228-4C31-81ED-5F9187C7F2A9}"
     },
+    {
+      "name": "grpc_csharp_ext",
+      "build": "all",
+      "language": "c",
+      "src": [
+        "src/csharp/ext/grpc_csharp_ext.c"
+      ],
+      "deps": [
+        "gpr",
+        "grpc"
+      ]
+    },
     {
       "name": "grpc_test_util",
       "build": "private",
@@ -398,11 +423,11 @@
         "src/cpp/client/create_channel.cc",
         "src/cpp/client/credentials.cc",
         "src/cpp/client/internal_stub.cc",
+        "src/cpp/common/completion_queue.cc",
         "src/cpp/common/rpc_method.cc",
         "src/cpp/proto/proto_utils.cc",
         "src/cpp/server/async_server.cc",
         "src/cpp/server/async_server_context.cc",
-        "src/cpp/server/completion_queue.cc",
         "src/cpp/server/server.cc",
         "src/cpp/server/server_builder.cc",
         "src/cpp/server/server_context_impl.cc",
@@ -447,18 +472,6 @@
         "grpc",
         "gpr"
       ]
-    },
-    {
-      "name": "grpc_csharp_ext",
-      "build": "all",
-      "language": "c",
-      "deps": [
-        "gpr",
-        "grpc"
-      ],
-      "src": [
-        "src/csharp/ext/grpc_csharp_ext.c"
-      ]
     }
   ],
   "targets": [
@@ -895,6 +908,30 @@
         "gpr"
       ]
     },
+    {
+      "name": "gpr_env_test",
+      "build": "test",
+      "language": "c",
+      "src": [
+        "test/core/support/env_test.c"
+      ],
+      "deps": [
+        "gpr_test_util",
+        "gpr"
+      ]
+    },
+    {
+      "name": "gpr_file_test",
+      "build": "test",
+      "language": "c",
+      "src": [
+        "test/core/support/file_test.c"
+      ],
+      "deps": [
+        "gpr_test_util",
+        "gpr"
+      ]
+    },
     {
       "name": "gpr_histogram_test",
       "build": "test",
@@ -1587,32 +1624,31 @@
       "run": false
     },
     {
-      "name": "tips_client",
+      "name": "qps_client",
       "build": "test",
       "language": "c++",
       "src": [
-        "examples/tips/main.cc"
+        "test/cpp/qps/qpstest.proto",
+        "test/cpp/qps/client.cc"
       ],
       "deps": [
-        "tips_client_lib",
         "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
         "grpc",
         "gpr_test_util",
         "gpr"
-      ],
+      ]
-      "run": false
     },
     {
-      "name": "tips_publisher_test",
+      "name": "qps_server",
       "build": "test",
       "language": "c++",
       "src": [
-        "examples/tips/publisher_test.cc"
+        "test/cpp/qps/qpstest.proto",
+        "test/cpp/qps/server.cc"
       ],
       "deps": [
-        "tips_client_lib",
         "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
@@ -1622,15 +1658,30 @@
       ]
     },
     {
-      "name": "tips_subscriber_test",
+      "name": "ruby_plugin",
+      "build": "protoc",
+      "language": "c++",
+      "headers": [
+        "src/compiler/cpp_generator.h",
+        "src/compiler/cpp_generator_helpers-inl.h",
+        "src/compiler/cpp_generator_map-inl.h",
+        "src/compiler/cpp_generator_string-inl.h"
+      ],
+      "src": [
+        "src/compiler/ruby_generator.cc",
+        "src/compiler/ruby_plugin.cc"
+      ],
+      "deps": [],
+      "secure": false
+    },
+    {
+      "name": "status_test",
       "build": "test",
       "language": "c++",
       "src": [
-        "examples/tips/subscriber_test.cc"
+        "test/cpp/util/status_test.cc"
       ],
       "deps": [
-        "tips_client_lib",
-        "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -1639,12 +1690,11 @@
       ]
     },
     {
-      "name": "qps_client",
+      "name": "sync_client_async_server_test",
       "build": "test",
       "language": "c++",
       "src": [
-        "test/cpp/qps/qpstest.proto",
+        "test/cpp/end2end/sync_client_async_server_test.cc"
-        "test/cpp/qps/client.cc"
       ],
       "deps": [
         "grpc++_test_util",
@@ -1656,15 +1706,13 @@
       ]
     },
     {
-      "name": "qps_server",
+      "name": "thread_pool_test",
       "build": "test",
       "language": "c++",
       "src": [
-        "test/cpp/qps/qpstest.proto",
+        "test/cpp/server/thread_pool_test.cc"
-        "test/cpp/qps/server.cc"
       ],
       "deps": [
-        "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -1673,45 +1721,32 @@
       ]
     },
     {
-      "name": "ruby_plugin",
+      "name": "tips_client",
-      "build": "protoc",
-      "language": "c++",
-      "headers": [
-        "src/compiler/cpp_generator.h",
-        "src/compiler/cpp_generator_helpers-inl.h",
-        "src/compiler/cpp_generator_map-inl.h",
-        "src/compiler/cpp_generator_string-inl.h"
-      ],
-      "src": [
-        "src/compiler/ruby_generator.cc",
-        "src/compiler/ruby_plugin.cc"
-      ],
-      "deps": [],
-      "secure": false
-    },
-    {
-      "name": "status_test",
       "build": "test",
       "language": "c++",
       "src": [
-        "test/cpp/util/status_test.cc"
+        "examples/tips/main.cc"
       ],
       "deps": [
+        "tips_client_lib",
+        "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
         "grpc",
         "gpr_test_util",
         "gpr"
-      ]
+      ],
+      "run": false
     },
     {
-      "name": "sync_client_async_server_test",
+      "name": "tips_publisher_test",
       "build": "test",
       "language": "c++",
       "src": [
-        "test/cpp/end2end/sync_client_async_server_test.cc"
+        "examples/tips/publisher_test.cc"
       ],
       "deps": [
+        "tips_client_lib",
         "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
@@ -1721,13 +1756,15 @@
       ]
     },
     {
-      "name": "thread_pool_test",
+      "name": "tips_subscriber_test",
       "build": "test",
       "language": "c++",
       "src": [
-        "test/cpp/server/thread_pool_test.cc"
+        "examples/tips/subscriber_test.cc"
       ],
       "deps": [
+        "tips_client_lib",
+        "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
         "grpc",

examples/tips/publisher_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <google/protobuf/stubs/common.h>
+
 #include <grpc++/channel_arguments.h>
 #include <grpc++/channel_interface.h>
 #include <grpc++/client_context.h>

examples/tips/subscriber_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <google/protobuf/stubs/common.h>
+
 #include <grpc++/channel_arguments.h>
 #include <grpc++/channel_interface.h>
 #include <grpc++/client_context.h>

include/grpc++/credentials.h

@@ -66,14 +66,13 @@ class Credentials final {
 
 // Options used to build SslCredentials
 // pem_roots_cert is the buffer containing the PEM encoding of the server root
-// certificates. This parameter cannot be empty.
+// certificates. If this parameter is empty, the default roots will be used.
 // pem_private_key is the buffer containing the PEM encoding of the client's
 // private key. This parameter can be empty if the client does not have a
 // private key.
 // pem_cert_chain is the buffer containing the PEM encoding of the client's
 // certificate chain. This parameter can be empty if the client does not have
 // a certificate chain.
-// TODO(jboeuf) Change it to point to a file.
 struct SslCredentialsOptions {
   grpc::string pem_root_certs;
   grpc::string pem_private_key;

include/grpc/grpc.h

@@ -177,23 +177,22 @@ void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader);
 
 /* A single metadata element */
 typedef struct grpc_metadata {
-  char *key;
+  const char *key;
-  char *value;
+  const char *value;
   size_t value_length;
 } grpc_metadata;
 
 typedef enum grpc_completion_type {
-  GRPC_QUEUE_SHUTDOWN,  /* Shutting down */
+  GRPC_QUEUE_SHUTDOWN,       /* Shutting down */
-  GRPC_READ,            /* A read has completed */
+  GRPC_OP_COMPLETE,          /* operation completion */
-  GRPC_INVOKE_ACCEPTED, /* An invoke call has been accepted by flow
+  GRPC_READ,                 /* A read has completed */
-                           control */
+  GRPC_WRITE_ACCEPTED,       /* A write has been accepted by
-  GRPC_WRITE_ACCEPTED, /* A write has been accepted by
+                                flow control */
-                          flow control */
   GRPC_FINISH_ACCEPTED,      /* writes_done or write_status has been accepted */
   GRPC_CLIENT_METADATA_READ, /* The metadata array sent by server received at
                                 client */
-  GRPC_FINISHED, /* An RPC has finished. The event contains status.
+  GRPC_FINISHED,             /* An RPC has finished. The event contains status.
-                    On the server this will be OK or Cancelled. */
+                                On the server this will be OK or Cancelled. */
   GRPC_SERVER_RPC_NEW,       /* A new RPC has arrived at the server */
   GRPC_SERVER_SHUTDOWN,      /* The server has finished shutting down */
   GRPC_COMPLETION_DO_NOT_USE /* must be last, forces users to include
@@ -213,6 +212,7 @@ typedef struct grpc_event {
     grpc_op_error write_accepted;
     grpc_op_error finish_accepted;
     grpc_op_error invoke_accepted;
+    grpc_op_error op_complete;
     struct {
       size_t count;
       grpc_metadata *elements;
@@ -233,6 +233,108 @@ typedef struct grpc_event {
   } data;
 } grpc_event;
 
+typedef struct {
+  size_t count;
+  size_t capacity;
+  grpc_metadata *metadata;
+} grpc_metadata_array;
+
+void grpc_metadata_array_init(grpc_metadata_array *array);
+void grpc_metadata_array_destroy(grpc_metadata_array *array);
+
+typedef struct {
+  char *method;
+  size_t method_capacity;
+  char *host;
+  size_t host_capacity;
+  gpr_timespec deadline;
+} grpc_call_details;
+
+void grpc_call_details_init(grpc_call_details *details);
+void grpc_call_details_destroy(grpc_call_details *details);
+
+typedef enum {
+  /* Send initial metadata: one and only one instance MUST be sent for each call,
+     unless the call was cancelled - in which case this can be skipped */
+  GRPC_OP_SEND_INITIAL_METADATA = 0,
+  /* Send a message: 0 or more of these operations can occur for each call */
+  GRPC_OP_SEND_MESSAGE,
+  /* Send a close from the server: one and only one instance MUST be sent from the client,
+     unless the call was cancelled - in which case this can be skipped */
+  GRPC_OP_SEND_CLOSE_FROM_CLIENT,
+  /* Send status from the server: one and only one instance MUST be sent from the server
+     unless the call was cancelled - in which case this can be skipped */
+  GRPC_OP_SEND_STATUS_FROM_SERVER,
+  /* Receive initial metadata: one and only one MUST be made on the client, must
+     not be made on the server */
+  GRPC_OP_RECV_INITIAL_METADATA,
+  /* Receive a message: 0 or more of these operations can occur for each call */
+  GRPC_OP_RECV_MESSAGE,
+  /* Receive status on the client: one and only one must be made on the client */
+  GRPC_OP_RECV_STATUS_ON_CLIENT,
+  /* Receive status on the server: one and only one must be made on the server */
+  GRPC_OP_RECV_CLOSE_ON_SERVER
+} grpc_op_type;
+
+/* Operation data: one field for each op type (except SEND_CLOSE_FROM_CLIENT which has
+   no arguments) */
+typedef struct grpc_op {
+  grpc_op_type op;
+  union {
+    struct {
+      size_t count;
+      const grpc_metadata *metadata;
+    } send_initial_metadata;
+    grpc_byte_buffer *send_message;
+    struct {
+      size_t trailing_metadata_count;
+      grpc_metadata *trailing_metadata;
+      grpc_status_code status;
+      const char *status_details;
+    } send_status_from_server;
+    /* ownership of the array is with the caller, but ownership of the elements
+       stays with the call object (ie key, value members are owned by the call
+       object, recv_initial_metadata->array is owned by the caller).
+       After the operation completes, call grpc_metadata_array_destroy on this
+       value, or reuse it in a future op. */
+    grpc_metadata_array *recv_initial_metadata;
+    grpc_byte_buffer **recv_message;
+    struct {
+      /* ownership of the array is with the caller, but ownership of the elements
+         stays with the call object (ie key, value members are owned by the call
+         object, trailing_metadata->array is owned by the caller).
+         After the operation completes, call grpc_metadata_array_destroy on this
+         value, or reuse it in a future op. */
+      grpc_metadata_array *trailing_metadata;
+      grpc_status_code *status;
+      /* status_details is a buffer owned by the application before the op completes
+         and after the op has completed. During the operation status_details may be
+         reallocated to a size larger than *status_details_capacity, in which case
+         *status_details_capacity will be updated with the new array capacity.
+
+         Pre-allocating space:
+         size_t my_capacity = 8;
+         char *my_details = gpr_malloc(my_capacity);
+         x.status_details = &my_details;
+         x.status_details_capacity = &my_capacity; 
+
+         Not pre-allocating space:
+         size_t my_capacity = 0;
+         char *my_details = NULL;
+         x.status_details = &my_details;
+         x.status_details_capacity = &my_capacity; 
+
+         After the call:
+         gpr_free(my_details); */
+      char **status_details;
+      size_t *status_details_capacity;
+    } recv_status_on_client;
+    struct {
+      int *cancelled;
+    } recv_close_on_server;
+  } data;
+} grpc_op;
+
 /* Initialize the grpc library */
 void grpc_init(void);
 
@@ -279,6 +381,22 @@ grpc_call *grpc_channel_create_call_old(grpc_channel *channel,
                                         const char *method, const char *host,
                                         gpr_timespec deadline);
 
+/* Create a call given a grpc_channel, in order to call 'method'. The request
+   is not sent until grpc_call_invoke is called. All completions are sent to
+   'completion_queue'. */
+grpc_call *grpc_channel_create_call(grpc_channel *channel,
+                                    grpc_completion_queue *completion_queue,
+                                    const char *method, const char *host,
+                                    gpr_timespec deadline);
+
+/* Start a batch of operations defined in the array ops; when complete, post a
+   completion of type 'tag' to the completion queue bound to the call. 
+   The order of ops specified in the batch has no significance.
+   Only one operation of each type can be active at once in any given
+   batch. */
+grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
+                                      size_t nops, void *tag);
+
 /* Create a client channel */
 grpc_channel *grpc_channel_create(const char *target,
                                   const grpc_channel_args *args);
@@ -419,6 +537,11 @@ void grpc_call_destroy(grpc_call *call);
 grpc_call_error grpc_server_request_call_old(grpc_server *server,
                                              void *tag_new);
 
+grpc_call_error grpc_server_request_call(
+    grpc_server *server, grpc_call **call, grpc_call_details *details,
+    grpc_metadata_array *request_metadata,
+    grpc_completion_queue *completion_queue, void *tag_new);
+
 /* Create a server */
 grpc_server *grpc_server_create(grpc_completion_queue *cq,
                                 const grpc_channel_args *args);

include/grpc/grpc_security.h

@@ -54,6 +54,12 @@ void grpc_credentials_release(grpc_credentials *creds);
 /* Creates default credentials. */
 grpc_credentials *grpc_default_credentials_create(void);
 
+/* Environment variable that points to the default SSL roots file. This file
+   must be a PEM encoded file with all the roots such as the one that can be
+   downloaded from https://pki.google.com/roots.pem.  */
+#define GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR \
+  "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH"
+
 /* Object that holds a private key / certificate chain pair in PEM format. */
 typedef struct {
   /* private_key is the NULL-terminated string containing the PEM encoding of

include/grpc/support/port_platform.h

@@ -63,6 +63,8 @@
 #define GPR_POSIX_SOCKET 1
 #define GPR_POSIX_SOCKETADDR 1
 #define GPR_POSIX_SOCKETUTILS 1
+#define GPR_POSIX_ENV 1
+#define GPR_POSIX_FILE 1
 #define GPR_POSIX_STRING 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
@@ -76,6 +78,8 @@
 #define GPR_LINUX_EVENTFD 1
 #define GPR_POSIX_SOCKET 1
 #define GPR_POSIX_SOCKETADDR 1
+#define GPR_LINUX_ENV 1
+#define GPR_POSIX_FILE 1
 #define GPR_POSIX_STRING 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
@@ -95,6 +99,8 @@
 #define GPR_POSIX_SOCKET 1
 #define GPR_POSIX_SOCKETADDR 1
 #define GPR_POSIX_SOCKETUTILS 1
+#define GPR_POSIX_ENV 1
+#define GPR_POSIX_FILE 1
 #define GPR_POSIX_STRING 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1

src/core/channel/channel_stack.c

@@ -210,6 +210,7 @@ void grpc_call_element_recv_metadata(grpc_call_element *cur_elem,
   metadata_op.dir = GRPC_CALL_UP;
   metadata_op.done_cb = do_nothing;
   metadata_op.user_data = NULL;
+  metadata_op.flags = 0;
   metadata_op.data.metadata = mdelem;
   grpc_call_next_op(cur_elem, &metadata_op);
 }
@@ -221,6 +222,7 @@ void grpc_call_element_send_metadata(grpc_call_element *cur_elem,
   metadata_op.dir = GRPC_CALL_DOWN;
   metadata_op.done_cb = do_nothing;
   metadata_op.user_data = NULL;
+  metadata_op.flags = 0;
   metadata_op.data.metadata = mdelem;
   grpc_call_next_op(cur_elem, &metadata_op);
 }
@@ -231,14 +233,16 @@ void grpc_call_element_send_cancel(grpc_call_element *cur_elem) {
   cancel_op.dir = GRPC_CALL_DOWN;
   cancel_op.done_cb = do_nothing;
   cancel_op.user_data = NULL;
+  cancel_op.flags = 0;
   grpc_call_next_op(cur_elem, &cancel_op);
 }
 
 void grpc_call_element_send_finish(grpc_call_element *cur_elem) {
-  grpc_call_op cancel_op;
+  grpc_call_op finish_op;
-  cancel_op.type = GRPC_SEND_FINISH;
+  finish_op.type = GRPC_SEND_FINISH;
-  cancel_op.dir = GRPC_CALL_DOWN;
+  finish_op.dir = GRPC_CALL_DOWN;
-  cancel_op.done_cb = do_nothing;
+  finish_op.done_cb = do_nothing;
-  cancel_op.user_data = NULL;
+  finish_op.user_data = NULL;
-  grpc_call_next_op(cur_elem, &cancel_op);
+  finish_op.flags = 0;
+  grpc_call_next_op(cur_elem, &finish_op);
 }

src/core/channel/client_channel.c

@@ -210,11 +210,30 @@ static void remove_waiting_child(channel_data *chand, call_data *calld) {
   chand->waiting_child_count = new_count;
 }
 
+static void send_up_cancelled_ops(grpc_call_element *elem) {
+  grpc_call_op finish_op;
+  channel_data *chand = elem->channel_data;
+  /* send up a synthesized status */
+  finish_op.type = GRPC_RECV_METADATA;
+  finish_op.dir = GRPC_CALL_UP;
+  finish_op.flags = 0;
+  finish_op.data.metadata = grpc_mdelem_ref(chand->cancel_status);
+  finish_op.done_cb = do_nothing;
+  finish_op.user_data = NULL;
+  grpc_call_next_op(elem, &finish_op);
+  /* send up a finish */
+  finish_op.type = GRPC_RECV_FINISH;
+  finish_op.dir = GRPC_CALL_UP;
+  finish_op.flags = 0;
+  finish_op.done_cb = do_nothing;
+  finish_op.user_data = NULL;
+  grpc_call_next_op(elem, &finish_op);
+}
+
 static void cancel_rpc(grpc_call_element *elem, grpc_call_op *op) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
   grpc_call_element *child_elem;
-  grpc_call_op finish_op;
 
   gpr_mu_lock(&chand->mu);
   switch (calld->state) {
@@ -225,27 +244,16 @@ static void cancel_rpc(grpc_call_element *elem, grpc_call_op *op) {
       return; /* early out */
     case CALL_WAITING:
       remove_waiting_child(chand, calld);
+      calld->state = CALL_CANCELLED;
+      gpr_mu_unlock(&chand->mu);
+      send_up_cancelled_ops(elem);
       calld->s.waiting.on_complete(calld->s.waiting.on_complete_user_data,
                                    GRPC_OP_ERROR);
-    /* fallthrough intended */
+      return; /* early out */
     case CALL_CREATED:
       calld->state = CALL_CANCELLED;
       gpr_mu_unlock(&chand->mu);
-      /* send up a synthesized status */
+      send_up_cancelled_ops(elem);
-      finish_op.type = GRPC_RECV_METADATA;
-      finish_op.dir = GRPC_CALL_UP;
-      finish_op.flags = 0;
-      finish_op.data.metadata = grpc_mdelem_ref(chand->cancel_status);
-      finish_op.done_cb = do_nothing;
-      finish_op.user_data = NULL;
-      grpc_call_next_op(elem, &finish_op);
-      /* send up a finish */
-      finish_op.type = GRPC_RECV_FINISH;
-      finish_op.dir = GRPC_CALL_UP;
-      finish_op.flags = 0;
-      finish_op.done_cb = do_nothing;
-      finish_op.user_data = NULL;
-      grpc_call_next_op(elem, &finish_op);
       return; /* early out */
     case CALL_CANCELLED:
       gpr_mu_unlock(&chand->mu);
@@ -298,6 +306,7 @@ static void channel_op(grpc_channel_element *elem,
                        grpc_channel_element *from_elem, grpc_channel_op *op) {
   channel_data *chand = elem->channel_data;
   grpc_child_channel *child_channel;
+  grpc_channel_op rop;
   GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
 
   switch (op->type) {
@@ -323,6 +332,10 @@ static void channel_op(grpc_channel_element *elem,
       if (child_channel) {
         grpc_child_channel_destroy(child_channel, 1);
       }
+      /* fake a transport closed to satisfy the refcounting in client */
+      rop.type = GRPC_TRANSPORT_CLOSED;
+      rop.dir = GRPC_CALL_UP;
+      grpc_channel_next_op(elem, &rop);
       break;
     case GRPC_TRANSPORT_GOAWAY:
       /* receiving goaway: if it's from our active child, drop the active child;

src/core/channel/connected_channel.c

@@ -298,10 +298,6 @@ static void recv_error(channel_data *chand, call_data *calld, int line,
 
 static void do_nothing(void *calldata, grpc_op_error error) {}
 
-static void done_message(void *user_data, grpc_op_error error) {
-  grpc_byte_buffer_destroy(user_data);
-}
-
 static void finish_message(channel_data *chand, call_data *calld) {
   grpc_call_element *elem = calld->elem;
   grpc_call_op call_op;
@@ -309,9 +305,9 @@ static void finish_message(channel_data *chand, call_data *calld) {
   call_op.flags = 0;
   /* if we got all the bytes for this message, call up the stack */
   call_op.type = GRPC_RECV_MESSAGE;
-  call_op.done_cb = done_message;
+  call_op.done_cb = do_nothing;
   /* TODO(ctiller): this could be a lot faster if coded directly */
-  call_op.user_data = call_op.data.message = grpc_byte_buffer_create(
+  call_op.data.message = grpc_byte_buffer_create(
       calld->incoming_message.slices, calld->incoming_message.count);
   gpr_slice_buffer_reset_and_unref(&calld->incoming_message);
 

src/core/channel/http_client_filter.c

@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2014, Google Inc.
  * All rights reserved.
  *
@@ -44,6 +43,7 @@ typedef struct channel_data {
   grpc_mdelem *method;
   grpc_mdelem *scheme;
   grpc_mdelem *content_type;
+  grpc_mdelem *status;
 } channel_data;
 
 /* used to silence 'variable not used' warnings */
@@ -86,6 +86,18 @@ static void call_op(grpc_call_element *elem, grpc_call_element *from_elem,
       grpc_call_element_send_metadata(elem, grpc_mdelem_ref(channeld->content_type));
       grpc_call_next_op(elem, op);
       break;
+    case GRPC_RECV_METADATA:
+      if (op->data.metadata == channeld->status) {
+        grpc_mdelem_unref(op->data.metadata);
+        op->done_cb(op->user_data, GRPC_OP_OK);
+      } else if (op->data.metadata->key == channeld->status->key) {
+        grpc_mdelem_unref(op->data.metadata);
+        op->done_cb(op->user_data, GRPC_OP_OK);
+        grpc_call_element_send_cancel(elem);
+      } else {
+        grpc_call_next_op(elem, op);
+      }
+      break;
     default:
       /* pass control up or down the stack depending on op->dir */
       grpc_call_next_op(elem, op);
@@ -166,6 +178,7 @@ static void init_channel_elem(grpc_channel_element *elem,
       grpc_mdelem_from_strings(mdctx, ":scheme", scheme_from_args(args));
   channeld->content_type =
       grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc");
+  channeld->status = grpc_mdelem_from_strings(mdctx, ":status", "200");
 }
 
 /* Destructor for channel data */
@@ -177,6 +190,7 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
   grpc_mdelem_unref(channeld->method);
   grpc_mdelem_unref(channeld->scheme);
   grpc_mdelem_unref(channeld->content_type);
+  grpc_mdelem_unref(channeld->status);
 }
 
 const grpc_channel_filter grpc_http_client_filter = {

src/core/iomgr/alarm.c

@@ -335,10 +335,6 @@ static int run_some_expired_alarms(gpr_mu *drop_mu, gpr_timespec now,
 
     gpr_mu_unlock(&g_mu);
     gpr_mu_unlock(&g_checker_mu);
-  } else if (next) {
-    gpr_mu_lock(&g_mu);
-    *next = gpr_time_min(*next, g_shard_queue[0]->min_deadline);
-    gpr_mu_unlock(&g_mu);
   }
 
   if (n && drop_mu) {

src/core/iomgr/alarm_internal.h

@@ -39,6 +39,15 @@
 
 /* iomgr internal api for dealing with alarms */
 
+/* Check for alarms to be run, and run them. 
+   Return non zero if alarm callbacks were executed.
+   Drops drop_mu if it is non-null before executing callbacks.
+   If next is non-null, TRY to update *next with the next running alarm
+   IF that alarm occurs before *next current value.
+   *next is never guaranteed to be updated on any given execution; however,
+   with high probability at least one thread in the system will see an update
+   at any time slice. */
+
 int grpc_alarm_check(gpr_mu *drop_mu, gpr_timespec now, gpr_timespec *next);
 
 void grpc_alarm_list_init(gpr_timespec now);

src/core/iomgr/pollset_kick.c

@@ -48,49 +48,49 @@
 /* This implementation is based on a freelist of wakeup fds, with extra logic to
  * handle kicks while there is no attached fd. */
 
+/* TODO(klempner): Autosize this, and consider providing a way to disable the
+ * cap entirely on systems with large fd limits */
 #define GRPC_MAX_CACHED_WFDS 50
-#define GRPC_WFD_LOW_WATERMARK 25
 
 static grpc_kick_fd_info *fd_freelist = NULL;
 static int fd_freelist_count = 0;
 static gpr_mu fd_freelist_mu;
 
 static grpc_kick_fd_info *allocate_wfd(void) {
-  grpc_kick_fd_info *info;
+  grpc_kick_fd_info *info = NULL;
   gpr_mu_lock(&fd_freelist_mu);
   if (fd_freelist != NULL) {
     info = fd_freelist;
     fd_freelist = fd_freelist->next;
     --fd_freelist_count;
-  } else {
+  }
+  gpr_mu_unlock(&fd_freelist_mu);
+  if (info == NULL) {
     info = gpr_malloc(sizeof(*info));
     grpc_wakeup_fd_create(&info->wakeup_fd);
     info->next = NULL;
   }
-  gpr_mu_unlock(&fd_freelist_mu);
   return info;
 }
 
-static void destroy_wfd(void) {
+static void destroy_wfd(grpc_kick_fd_info* wfd) {
-  /* assumes fd_freelist_mu is held */
+  grpc_wakeup_fd_destroy(&wfd->wakeup_fd);
-  grpc_kick_fd_info *current = fd_freelist;
+  gpr_free(wfd);
-  fd_freelist = fd_freelist->next;
-  fd_freelist_count--;
-  grpc_wakeup_fd_destroy(&current->wakeup_fd);
-  gpr_free(current);
 }
 
 static void free_wfd(grpc_kick_fd_info *fd_info) {
   gpr_mu_lock(&fd_freelist_mu);
-  fd_info->next = fd_freelist;
+  if (fd_freelist_count < GRPC_MAX_CACHED_WFDS) {
-  fd_freelist = fd_info;
+    fd_info->next = fd_freelist;
-  fd_freelist_count++;
+    fd_freelist = fd_info;
-  if (fd_freelist_count > GRPC_MAX_CACHED_WFDS) {
+    fd_freelist_count++;
-    while (fd_freelist_count > GRPC_WFD_LOW_WATERMARK) {
+    fd_info = NULL;
-      destroy_wfd();
-    }
   }
   gpr_mu_unlock(&fd_freelist_mu);
+
+  if (fd_info) {
+    destroy_wfd(fd_info);
+  }
 }
 
 void grpc_pollset_kick_init(grpc_pollset_kick_state *kick_state) {
@@ -148,6 +148,11 @@ void grpc_pollset_kick_global_init(void) {
 }
 
 void grpc_pollset_kick_global_destroy(void) {
+  while (fd_freelist != NULL) {
+    grpc_kick_fd_info *current = fd_freelist;
+    fd_freelist = fd_freelist->next;
+    destroy_wfd(current);
+  }
   grpc_wakeup_fd_global_destroy();
   gpr_mu_destroy(&fd_freelist_mu);
 }

src/core/iomgr/pollset_posix.c

@@ -80,9 +80,7 @@ void grpc_pollset_kick(grpc_pollset *p) {
   }
 }
 
-void grpc_pollset_force_kick(grpc_pollset *p) {
+void grpc_pollset_force_kick(grpc_pollset *p) { grpc_pollset_kick_kick(&p->kick_state); }
-  grpc_pollset_kick_kick(&p->kick_state);
-}
 
 /* global state management */
 

src/core/iomgr/resolve_address.c

@@ -31,7 +31,9 @@
  *
  */
 
+#ifndef _POSIX_SOURCE
 #define _POSIX_SOURCE
+#endif
 
 #include "src/core/iomgr/sockaddr.h"
 #include "src/core/iomgr/resolve_address.h"

src/core/iomgr/socket_utils_linux.c

@@ -31,7 +31,9 @@
  *
  */
 
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_LINUX

src/core/iomgr/socket_utils_posix.c

@@ -35,7 +35,6 @@
 
 #ifdef GPR_POSIX_SOCKETUTILS
 
-#define _BSD_SOURCE
 #include "src/core/iomgr/socket_utils_posix.h"
 
 #include <fcntl.h>

src/core/iomgr/tcp_server_posix.c

@@ -31,11 +31,15 @@
  *
  */
 
+/* FIXME: "posix" files shouldn't be depending on _GNU_SOURCE */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_POSIX_SOCKET
 
-#define _GNU_SOURCE
 #include "src/core/iomgr/tcp_server.h"
 
 #include <limits.h>

src/core/security/credentials.c

@@ -216,14 +216,10 @@ static void ssl_copy_key_material(const char *input, unsigned char **output,
 static void ssl_build_config(const char *pem_root_certs,
                              grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
                              grpc_ssl_config *config) {
-  if (pem_root_certs == NULL) {
+  if (pem_root_certs != NULL) {
-    /* TODO(jboeuf): Get them from the environment. */
-    gpr_log(GPR_ERROR, "Default SSL roots not yet implemented.");
-  } else {
     ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
                           &config->pem_root_certs_size);
   }
-
   if (pem_key_cert_pair != NULL) {
     GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
     GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);

src/core/security/security_context.c

@@ -39,6 +39,8 @@
 #include "src/core/channel/http_client_filter.h"
 #include "src/core/security/credentials.h"
 #include "src/core/security/secure_endpoint.h"
+#include "src/core/support/env.h"
+#include "src/core/support/file.h"
 #include "src/core/support/string.h"
 #include "src/core/surface/lame_client.h"
 #include "src/core/transport/chttp2/alpn.h"
@@ -319,6 +321,28 @@ static grpc_security_context_vtable ssl_channel_vtable = {
 static grpc_security_context_vtable ssl_server_vtable = {
     ssl_server_destroy, ssl_server_create_handshaker, ssl_server_check_peer};
 
+static gpr_slice default_pem_root_certs;
+
+static void init_default_pem_root_certs(void) {
+  char *default_root_certs_path =
+      gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
+  if (default_root_certs_path == NULL) {
+    default_pem_root_certs = gpr_empty_slice();
+  } else {
+    default_pem_root_certs = gpr_load_file(default_root_certs_path, NULL);
+    gpr_free(default_root_certs_path);
+  }
+}
+
+static size_t get_default_pem_roots(const unsigned char **pem_root_certs) {
+  /* TODO(jboeuf@google.com): Maybe revisit the approach which consists in
+     loading all the roots once for the lifetime of the process. */
+  static gpr_once once = GPR_ONCE_INIT;
+  gpr_once_init(&once, init_default_pem_root_certs);
+  *pem_root_certs = GPR_SLICE_START_PTR(default_pem_root_certs);
+  return GPR_SLICE_LENGTH(default_pem_root_certs);
+}
+
 grpc_security_status grpc_ssl_channel_security_context_create(
     grpc_credentials *request_metadata_creds, const grpc_ssl_config *config,
     const char *secure_peer_name, grpc_channel_security_context **ctx) {
@@ -330,6 +354,8 @@ grpc_security_status grpc_ssl_channel_security_context_create(
   tsi_result result = TSI_OK;
   grpc_ssl_channel_security_context *c;
   size_t i;
+  const unsigned char *pem_root_certs;
+  size_t pem_root_certs_size;
 
   for (i = 0; i < num_alpn_protocols; i++) {
     alpn_protocol_strings[i] =
@@ -338,9 +364,8 @@ grpc_security_status grpc_ssl_channel_security_context_create(
         strlen(grpc_chttp2_get_alpn_version_index(i));
   }
 
-  if (config == NULL || secure_peer_name == NULL ||
+  if (config == NULL || secure_peer_name == NULL) {
-      config->pem_root_certs == NULL) {
+    gpr_log(GPR_ERROR, "An ssl channel needs a config and a secure name.");
-    gpr_log(GPR_ERROR, "An ssl channel needs a secure name and root certs.");
     goto error;
   }
   if (!check_request_metadata_creds(request_metadata_creds)) {
@@ -357,11 +382,20 @@ grpc_security_status grpc_ssl_channel_security_context_create(
   if (secure_peer_name != NULL) {
     c->secure_peer_name = gpr_strdup(secure_peer_name);
   }
+  if (config->pem_root_certs == NULL) {
+    pem_root_certs_size = get_default_pem_roots(&pem_root_certs);
+    if (pem_root_certs == NULL || pem_root_certs_size == 0) {
+      gpr_log(GPR_ERROR, "Could not get default pem root certs.");
+      goto error;
+    }
+  } else {
+    pem_root_certs = config->pem_root_certs;
+    pem_root_certs_size = config->pem_root_certs_size;
+  }
   result = tsi_create_ssl_client_handshaker_factory(
       config->pem_private_key, config->pem_private_key_size,
-      config->pem_cert_chain, config->pem_cert_chain_size,
+      config->pem_cert_chain, config->pem_cert_chain_size, pem_root_certs,
-      config->pem_root_certs, config->pem_root_certs_size,
+      pem_root_certs_size, GRPC_SSL_CIPHER_SUITES, alpn_protocol_strings,
-      GRPC_SSL_CIPHER_SUITES, alpn_protocol_strings,
       alpn_protocol_string_lengths, num_alpn_protocols, &c->handshaker_factory);
   if (result != TSI_OK) {
     gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.",

src/core/statistics/census_rpc_stats.c

@@ -141,7 +141,7 @@ static void record_stats(census_ht* store, census_op_id op_id,
                          const census_rpc_stats* stats) {
   gpr_mu_lock(&g_mu);
   if (store != NULL) {
-    trace_obj* trace = NULL;
+    census_trace_obj* trace = NULL;
     census_internal_lock_trace_store();
     trace = census_get_trace_obj_locked(op_id);
     if (trace != NULL) {

src/core/statistics/census_tracing.c

@@ -32,38 +32,22 @@
  */
 
 #include "src/core/statistics/census_interface.h"
+#include "src/core/statistics/census_tracing.h"
 
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/statistics/census_rpc_stats.h"
 #include "src/core/statistics/hash_table.h"
 #include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/sync.h>
-#include <grpc/support/time.h>
+
-
+void census_trace_obj_destroy(census_trace_obj* obj) {
-/* Struct for a trace annotation. */
+  census_trace_annotation* p = obj->annotations;
-typedef struct annotation {
-  gpr_timespec ts;                            /* timestamp of the annotation */
-  char txt[CENSUS_MAX_ANNOTATION_LENGTH + 1]; /* actual txt annotation */
-  struct annotation* next;
-} annotation;
-
-typedef struct trace_obj {
-  census_op_id id;
-  gpr_timespec ts;
-  census_rpc_stats rpc_stats;
-  char* method;
-  annotation* annotations;
-} trace_obj;
-
-static void trace_obj_destroy(trace_obj* obj) {
-  annotation* p = obj->annotations;
   while (p != NULL) {
-    annotation* next = p->next;
+    census_trace_annotation* next = p->next;
     gpr_free(p);
     p = next;
   }
@@ -71,7 +55,9 @@ static void trace_obj_destroy(trace_obj* obj) {
   gpr_free(obj);
 }
 
-static void delete_trace_obj(void* obj) { trace_obj_destroy((trace_obj*)obj); }
+static void delete_trace_obj(void* obj) {
+  census_trace_obj_destroy((census_trace_obj*)obj);
+}
 
 static const census_ht_option ht_opt = {
     CENSUS_HT_UINT64 /* key type*/, 571 /* n_of_buckets */, NULL /* hash */,
@@ -103,8 +89,8 @@ static void init_mutex_once(void) {
 census_op_id census_tracing_start_op(void) {
   gpr_mu_lock(&g_mu);
   {
-    trace_obj* ret = (trace_obj*)gpr_malloc(sizeof(trace_obj));
+    census_trace_obj* ret = gpr_malloc(sizeof(census_trace_obj));
-    memset(ret, 0, sizeof(trace_obj));
+    memset(ret, 0, sizeof(census_trace_obj));
     g_id++;
     memcpy(&ret->id, &g_id, sizeof(census_op_id));
     ret->rpc_stats.cnt = 1;
@@ -118,7 +104,7 @@ census_op_id census_tracing_start_op(void) {
 
 int census_add_method_tag(census_op_id op_id, const char* method) {
   int ret = 0;
-  trace_obj* trace = NULL;
+  census_trace_obj* trace = NULL;
   gpr_mu_lock(&g_mu);
   trace = census_ht_find(g_trace_store, op_id_as_key(&op_id));
   if (trace == NULL) {
@@ -131,11 +117,11 @@ int census_add_method_tag(census_op_id op_id, const char* method) {
 }
 
 void census_tracing_print(census_op_id op_id, const char* anno_txt) {
-  trace_obj* trace = NULL;
+  census_trace_obj* trace = NULL;
   gpr_mu_lock(&g_mu);
   trace = census_ht_find(g_trace_store, op_id_as_key(&op_id));
   if (trace != NULL) {
-    annotation* anno = gpr_malloc(sizeof(annotation));
+    census_trace_annotation* anno = gpr_malloc(sizeof(census_trace_annotation));
     anno->ts = gpr_now();
     {
       char* d = anno->txt;
@@ -153,7 +139,7 @@ void census_tracing_print(census_op_id op_id, const char* anno_txt) {
 }
 
 void census_tracing_end_op(census_op_id op_id) {
-  trace_obj* trace = NULL;
+  census_trace_obj* trace = NULL;
   gpr_mu_lock(&g_mu);
   trace = census_ht_find(g_trace_store, op_id_as_key(&op_id));
   if (trace != NULL) {
@@ -196,14 +182,58 @@ void census_internal_lock_trace_store(void) { gpr_mu_lock(&g_mu); }
 
 void census_internal_unlock_trace_store(void) { gpr_mu_unlock(&g_mu); }
 
-trace_obj* census_get_trace_obj_locked(census_op_id op_id) {
+census_trace_obj* census_get_trace_obj_locked(census_op_id op_id) {
   if (g_trace_store == NULL) {
     gpr_log(GPR_ERROR, "Census trace store is not initialized.");
     return NULL;
   }
-  return (trace_obj*)census_ht_find(g_trace_store, op_id_as_key(&op_id));
+  return (census_trace_obj*)census_ht_find(g_trace_store, op_id_as_key(&op_id));
 }
 
-const char* census_get_trace_method_name(const trace_obj* trace) {
+const char* census_get_trace_method_name(const census_trace_obj* trace) {
-  return (const char*)trace->method;
+  return trace->method;
+}
+
+static census_trace_annotation* dup_annotation_chain(
+    census_trace_annotation* from) {
+  census_trace_annotation *ret = NULL;
+  census_trace_annotation **to = &ret;
+  for (; from != NULL; from = from->next) {
+    *to = gpr_malloc(sizeof(census_trace_annotation));
+    memcpy(*to, from, sizeof(census_trace_annotation));
+    to = &(*to)->next;
+  }
+  return ret;
+}
+
+static census_trace_obj* trace_obj_dup(census_trace_obj* from) {
+  census_trace_obj* to = NULL;
+  GPR_ASSERT(from != NULL);
+  to = gpr_malloc(sizeof(census_trace_obj));
+  to->id = from->id;
+  to->ts = from->ts;
+  to->rpc_stats = from->rpc_stats;
+  to->method = gpr_strdup(from->method);
+  to->annotations = dup_annotation_chain(from->annotations);
+  return to;
+}
+
+census_trace_obj** census_get_active_ops(int* num_active_ops) {
+  census_trace_obj** ret = NULL;
+  gpr_mu_lock(&g_mu);
+  if (g_trace_store != NULL) {
+    size_t n = 0;
+    census_ht_kv* all_kvs = census_ht_get_all_elements(g_trace_store, &n);
+    *num_active_ops = (int)n;
+    if (n != 0 ) {
+      size_t i = 0;
+      ret = gpr_malloc(sizeof(census_trace_obj *) * n);
+      for (i = 0; i < n; i++) {
+        ret[i] = trace_obj_dup((census_trace_obj*)all_kvs[i].v);
+      }
+    }
+    gpr_free(all_kvs);
+  }
+  gpr_mu_unlock(&g_mu);
+  return ret;
 }

src/core/statistics/census_tracing.h

@@ -34,12 +34,35 @@
 #ifndef __GRPC_INTERNAL_STATISTICS_CENSUS_TRACING_H_
 #define __GRPC_INTERNAL_STATISTICS_CENSUS_TRACING_H_
 
+#include <grpc/support/time.h>
+#include "src/core/statistics/census_rpc_stats.h"
+
+/* WARNING: The data structures and APIs provided by this file are for GRPC
+   library's internal use ONLY. They might be changed in backward-incompatible
+   ways and are not subject to any deprecation policy.
+   They are not recommended for external use.
+ */
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-/* Opaque structure for trace object */
+/* Struct for a trace annotation. */
-typedef struct trace_obj trace_obj;
+typedef struct census_trace_annotation {
+  gpr_timespec ts;                            /* timestamp of the annotation */
+  char txt[CENSUS_MAX_ANNOTATION_LENGTH + 1]; /* actual txt annotation */
+  struct census_trace_annotation* next;
+} census_trace_annotation;
+
+typedef struct census_trace_obj {
+  census_op_id id;
+  gpr_timespec ts;
+  census_rpc_stats rpc_stats;
+  char* method;
+  census_trace_annotation* annotations;
+} census_trace_obj;
+
+/* Deletes trace object. */
+void census_trace_obj_destroy(census_trace_obj* obj);
 
 /* Initializes trace store. This function is thread safe. */
 void census_tracing_init(void);
@@ -50,15 +73,21 @@ void census_tracing_shutdown(void);
 /* Gets trace obj corresponding to the input op_id. Returns NULL if trace store
    is not initialized or trace obj is not found. Requires trace store being
    locked before calling this function. */
-trace_obj* census_get_trace_obj_locked(census_op_id op_id);
+census_trace_obj* census_get_trace_obj_locked(census_op_id op_id);
 
 /* The following two functions acquire and release the trace store global lock.
    They are for census internal use only. */
 void census_internal_lock_trace_store(void);
 void census_internal_unlock_trace_store(void);
 
-/* Gets method tag name associated with the input trace object. */
+/* Gets method name associated with the input trace object. */
-const char* census_get_trace_method_name(const trace_obj* trace);
+const char* census_get_trace_method_name(const census_trace_obj* trace);
+
+/* Returns an array of pointers to trace objects of currently active operations
+   and fills in number of active operations. Returns NULL if there are no active
+   operations.
+   Caller owns the returned objects. */
+census_trace_obj** census_get_active_ops(int* num_active_ops);
 
 #ifdef __cplusplus
 }

src/core/support/env.h

@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __GRPC_SUPPORT_ENV_H__
+#define __GRPC_SUPPORT_ENV_H__
+
+#include <stdio.h>
+
+#include <grpc/support/slice.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Env utility functions */
+
+/* Gets the environment variable value with the specified name.
+   Returns a newly allocated string. It is the responsability of the caller to
+   gpr_free the return value if not NULL (which means that the environment
+   variable exists). */
+char *gpr_getenv(const char *name);
+
+/* Sets the the environment with the specified name to the specified value. */
+void gpr_setenv(const char *name, const char *value);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __GRPC_SUPPORT_ENV_H__ */

src/core/support/env_linux.c

@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* for secure_getenv. */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_LINUX_ENV
+
+#include "src/core/support/env.h"
+
+#include <stdlib.h>
+
+#include <grpc/support/log.h>
+
+#include "src/core/support/string.h"
+
+char *gpr_getenv(const char *name) {
+  char *result = secure_getenv(name);
+  return result == NULL ? result : gpr_strdup(result);
+}
+
+void gpr_setenv(const char *name, const char *value) {
+  int res = setenv(name, value, 1);
+  GPR_ASSERT(res == 0);
+}
+
+#endif /* GPR_LINUX_ENV */

src/core/support/env_posix.c

@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_POSIX_ENV
+
+#include "src/core/support/env.h"
+
+#include <stdlib.h>
+
+#include <grpc/support/log.h>
+
+#include "src/core/support/string.h"
+
+char *gpr_getenv(const char *name) {
+  char *result = getenv(name);
+  return result == NULL ? result : gpr_strdup(result);
+}
+
+void gpr_setenv(const char *name, const char *value) {
+  int res = setenv(name, value, 1);
+  GPR_ASSERT(res == 0);
+}
+
+#endif /* GPR_POSIX_ENV */

src/core/support/env_win32.c

@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_WIN32
+
+#include "src/core/support/env.h"
+
+#include <stdlib.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+char *gpr_getenv(const char *name) {
+  size_t required_size;
+  char *result = NULL;
+
+  getenv_s(&required_size, NULL, 0, name);
+  if (required_size == 0) return NULL;
+  result = gpr_malloc(required_size);
+  getenv_s(&required_size, result, required_size, name);
+  return result;
+}
+
+void gpr_setenv(const char *name, const char *value) {
+  errno_t res = _putenv_s(name, value);
+  GPR_ASSERT(res == 0);
+}
+
+#endif /* GPR_WIN32 */

src/core/support/file.c

@@ -0,0 +1,89 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/support/file.h"
+
+#include <errno.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/support/string.h"
+
+gpr_slice gpr_load_file(const char *filename, int *success) {
+  unsigned char *contents = NULL;
+  size_t contents_size = 0;
+  unsigned char buf[4096];
+  char *error_msg = NULL;
+  gpr_slice result = gpr_empty_slice();
+  FILE *file = fopen(filename, "rb");
+
+  if (file == NULL) {
+    gpr_asprintf(&error_msg, "Could not open file %s (error = %s).", filename,
+                 strerror(errno));
+    GPR_ASSERT(error_msg != NULL);
+    goto end;
+  }
+
+  while (1) {
+    size_t bytes_read = fread(buf, 1, sizeof(buf), file);
+    if (bytes_read > 0) {
+      contents = gpr_realloc(contents, contents_size + bytes_read);
+      memcpy(contents + contents_size, buf, bytes_read);
+      contents_size += bytes_read;
+    }
+    if (bytes_read < sizeof(buf)) {
+      if (ferror(file)) {
+        gpr_asprintf(&error_msg, "Error %s occured while reading file %s.",
+                     strerror(errno), filename);
+        GPR_ASSERT(error_msg != NULL);
+        goto end;
+      } else {
+        GPR_ASSERT(feof(file));
+        break;
+      }
+    }
+  }
+  if (success != NULL) *success = 1;
+  result = gpr_slice_new(contents, contents_size, gpr_free);
+
+end:
+  if (error_msg != NULL) {
+    gpr_log(GPR_ERROR, "%s", error_msg);
+    gpr_free(error_msg);
+    if (success != NULL) *success = 0;
+  }
+  if (file != NULL) fclose(file);
+  return result;
+}

src/core/support/file.h

@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __GRPC_SUPPORT_FILE_H__
+#define __GRPC_SUPPORT_FILE_H__
+
+#include <stdio.h>
+
+#include <grpc/support/slice.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* File utility functions */
+
+/* Loads the content of a file into a slice. The success parameter, if not NULL,
+   will be set to 1 in case of success and 0 in case of failure. */
+gpr_slice gpr_load_file(const char *filename, int *success);
+
+/* Creates a temporary file from a prefix.
+   If tmp_filename is not NULL, *tmp_filename is assigned the name of the
+   created file and it is the responsibility of the caller to gpr_free it
+   unless an error occurs in which case it will be set to NULL. */
+FILE *gpr_tmpfile(const char *prefix, char **tmp_filename);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __GRPC_SUPPORT_FILE_H__ */

src/core/support/file_posix.c

@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Posix code for gpr fdopen and mkstemp support. */
+
+#if !defined _POSIX_C_SOURCE || _POSIX_C_SOURCE < 200112L
+#undef _POSIX_C_SOURCE
+#define _POSIX_C_SOURCE 200112L
+#endif
+
+/* Don't know why I have to do this for mkstemp, looks like _POSIX_C_SOURCE
+   should be enough... */
+#ifndef _BSD_SOURCE
+#define _BSD_SOURCE
+#endif
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_POSIX_FILE
+
+#include "src/core/support/file.h"
+
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/support/string.h"
+
+FILE *gpr_tmpfile(const char *prefix, char **tmp_filename) {
+  FILE *result = NULL;
+  char *template;
+  int fd;
+
+  if (tmp_filename != NULL) *tmp_filename = NULL;
+
+  gpr_asprintf(&template, "/tmp/%s_XXXXXX", prefix);
+  GPR_ASSERT(template != NULL);
+
+  fd = mkstemp(template);
+  if (fd == -1) {
+    gpr_log(GPR_ERROR, "mkstemp failed for template %s with error %s.",
+            template, strerror(errno));
+    goto end;
+  }
+  result = fdopen(fd, "w+");
+  if (result == NULL) {
+    gpr_log(GPR_ERROR, "Could not open file %s from fd %d (error = %s).",
+            template, fd, strerror(errno));
+    unlink(template);
+    close(fd);
+    goto end;
+  }
+
+end:
+  if (result != NULL && tmp_filename != NULL) {
+    *tmp_filename = template;
+  } else {
+    gpr_free(template);
+  }
+  return result;
+}
+
+#endif /* GPR_POSIX_FILE */

src/core/support/file_win32.c

@@ -0,0 +1,83 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_WIN32
+
+#include <io.h>
+#include <stdio.h>
+#include <string.h>
+#include <tchar.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/support/file.h"
+#include "src/core/support/string_win32.h"
+
+FILE *gpr_tmpfile(const char *prefix, char **tmp_filename_out) {
+  FILE *result = NULL;
+  LPTSTR template_string = NULL;
+  TCHAR tmp_path[MAX_PATH];
+  TCHAR tmp_filename[MAX_PATH];
+  DWORD status;
+  UINT success;
+
+  if (tmp_filename_out != NULL) *tmp_filename_out = NULL;
+
+  /* Convert our prefix to TCHAR. */
+  template_string = gpr_char_to_tchar(prefix);
+  GPR_ASSERT(template_string);
+
+  /* Get the path to the best temporary folder available. */
+  status = GetTempPath(MAX_PATH, tmp_path);
+  if (status == 0 || status > MAX_PATH) goto end;
+
+  /* Generate a unique filename with our template + temporary path. */
+  success = GetTempFileName(tmp_path, template_string, 0, tmp_filename);
+  if (!success) goto end;
+
+  /* Open a file there. */
+  if (_tfopen_s(&result, tmp_filename, TEXT("wb+")) != 0) goto end;
+
+end:
+  if (result && tmp_filename) {
+    *tmp_filename_out = gpr_tchar_to_char(tmp_filename);
+  }
+
+  gpr_free(tmp_filename);
+  return result;
+}
+
+#endif /* GPR_WIN32 */

src/core/support/log_linux.c

@@ -31,8 +31,14 @@
  *
  */
 
+#ifndef _POSIX_SOURCE
 #define _POSIX_SOURCE
+#endif
+
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
+
 #include <grpc/support/port_platform.h>
 
 #ifdef GPR_LINUX

src/core/support/log_posix.c

@@ -31,11 +31,16 @@
  *
  */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined _POSIX_C_SOURCE || _POSIX_C_SOURCE < 200112L
+#undef _POSIX_C_SOURCE
 #define _POSIX_C_SOURCE 200112L
 #endif
 
+/* FIXME: "posix" files probably shouldn't depend on _GNU_SOURCE */
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
+
 #include <grpc/support/port_platform.h>
 
 #if defined(GPR_POSIX_LOG)

src/core/support/string_posix.c

@@ -33,7 +33,8 @@
 
 /* Posix code for gpr snprintf support. */
 
-#ifndef _POSIX_C_SOURCE
+#if !defined _POSIX_C_SOURCE || _POSIX_C_SOURCE < 200112L
+#undef _POSIX_C_SOURCE
 #define _POSIX_C_SOURCE 200112L
 #endif
 

src/core/support/string_win32.c

@@ -37,6 +37,7 @@
 
 #ifdef GPR_WIN32
 
+#include <windows.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
@@ -78,4 +79,32 @@ int gpr_asprintf(char **strp, const char *format, ...) {
   return -1;
 }
 
+#if defined UNICODE || defined _UNICODE
+LPTSTR gpr_char_to_tchar(LPCSTR input) {
+  LPTSTR ret;
+  int needed = MultiByteToWideChar(CP_UTF8, 0, input, -1, NULL, 0);
+  if (needed == 0) return NULL;
+  ret = gpr_malloc(needed * sizeof(TCHAR));
+  MultiByteToWideChar(CP_UTF8, 0, input, -1, ret, needed);
+  return ret;
+}
+
+LPSTR gpr_tchar_to_char(LPCTSTR input) {
+  LPSTR ret;
+  int needed = WideCharToMultiByte(CP_UTF8, 0, input, -1, NULL, 0, NULL, NULL);
+  if (needed == 0) return NULL;
+  ret = gpr_malloc(needed);
+  WideCharToMultiByte(CP_UTF8, 0, input, -1, ret, needed, NULL, NULL);
+  return ret;
+}
+#else
+char *gpr_tchar_to_char(LPTSTR input) {
+  return gpr_strdup(input);
+}
+
+char *gpr_char_to_tchar(LPTSTR input) {
+  return gpr_strdup(input);
+}
+#endif
+
 #endif /* GPR_WIN32 */

src/core/support/string_win32.h

@@ -0,0 +1,49 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __GRPC_SUPPORT_STRING_WIN32_H__
+#define __GRPC_SUPPORT_STRING_WIN32_H__
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_WIN32
+
+#include <windows.h>
+
+/* These allocate new strings using gpr_malloc to convert from and to utf-8. */
+LPTSTR gpr_char_to_tchar(LPCSTR input);
+LPSTR gpr_tchar_to_char(LPCTSTR input);
+
+#endif  /* GPR_WIN32 */
+
+#endif /* __GRPC_SUPPORT_STRING_WIN32_H__ */

src/core/surface/byte_buffer.c

@@ -61,6 +61,7 @@ grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb) {
 }
 
 void grpc_byte_buffer_destroy(grpc_byte_buffer *bb) {
+  if (!bb) return;
   switch (bb->type) {
     case GRPC_BB_SLICE_BUFFER:
       gpr_slice_buffer_destroy(&bb->data.slice_buffer);

src/core/surface/byte_buffer_queue.c

@@ -0,0 +1,91 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/surface/byte_buffer_queue.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/useful.h>
+
+static void bba_destroy(grpc_bbq_array *array, size_t start_pos) {
+  size_t i;
+  for (i = start_pos; i < array->count; i++) {
+    grpc_byte_buffer_destroy(array->data[i]);
+  }
+  gpr_free(array->data);
+}
+
+/* Append an operation to an array, expanding as needed */
+static void bba_push(grpc_bbq_array *a, grpc_byte_buffer *buffer) {
+  if (a->count == a->capacity) {
+    a->capacity = GPR_MAX(a->capacity * 2, 8);
+    a->data = gpr_realloc(a->data, sizeof(grpc_byte_buffer *) * a->capacity);
+  }
+  a->data[a->count++] = buffer;
+}
+
+void grpc_bbq_destroy(grpc_byte_buffer_queue *q) {
+  bba_destroy(&q->filling, 0);
+  bba_destroy(&q->draining, q->drain_pos);
+}
+
+int grpc_bbq_empty(grpc_byte_buffer_queue *q) {
+  return (q->drain_pos == q->draining.count && q->filling.count == 0);
+}
+
+void grpc_bbq_push(grpc_byte_buffer_queue *q, grpc_byte_buffer *buffer) {
+  bba_push(&q->filling, buffer);
+}
+
+void grpc_bbq_flush(grpc_byte_buffer_queue *q) {
+  grpc_byte_buffer *bb;
+  while ((bb = grpc_bbq_pop(q))) {
+    grpc_byte_buffer_destroy(bb);
+  }
+}
+
+grpc_byte_buffer *grpc_bbq_pop(grpc_byte_buffer_queue *q) {
+  grpc_bbq_array temp_array;
+
+  if (q->drain_pos == q->draining.count) {
+    if (q->filling.count == 0) {
+      return NULL;
+    }
+    q->draining.count = 0;
+    q->drain_pos = 0;
+    /* swap arrays */
+    temp_array = q->filling;
+    q->filling = q->draining;
+    q->draining = temp_array;
+  }
+
+  return q->draining.data[q->drain_pos++];
+}

src/core/surface/byte_buffer_queue.h

@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2014, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __GRPC_INTERNAL_SURFACE_BYTE_BUFFER_QUEUE_H__
+#define __GRPC_INTERNAL_SURFACE_BYTE_BUFFER_QUEUE_H__
+
+#include <grpc/byte_buffer.h>
+
+/* TODO(ctiller): inline an element or two into this struct to avoid per-call
+                  allocations */
+typedef struct {
+  grpc_byte_buffer **data;
+  size_t count;
+  size_t capacity;
+} grpc_bbq_array;
+
+/* should be initialized by zeroing memory */
+typedef struct {
+  size_t drain_pos;
+  grpc_bbq_array filling;
+  grpc_bbq_array draining;
+} grpc_byte_buffer_queue;
+
+void grpc_bbq_destroy(grpc_byte_buffer_queue *q);
+grpc_byte_buffer *grpc_bbq_pop(grpc_byte_buffer_queue *q);
+void grpc_bbq_flush(grpc_byte_buffer_queue *q);
+int grpc_bbq_empty(grpc_byte_buffer_queue *q);
+void grpc_bbq_push(grpc_byte_buffer_queue *q, grpc_byte_buffer *bb);
+
+#endif  /* __GRPC_INTERNAL_SURFACE_BYTE_BUFFER_QUEUE_H__ */

src/core/surface/call.h

@@ -38,28 +38,77 @@
 #include "src/core/channel/metadata_buffer.h"
 #include <grpc/grpc.h>
 
-grpc_call *grpc_call_create(grpc_channel *channel,
+/* Primitive operation types - grpc_op's get rewritten into these */
+typedef enum {
+  GRPC_IOREQ_RECV_INITIAL_METADATA,
+  GRPC_IOREQ_RECV_MESSAGE,
+  GRPC_IOREQ_RECV_TRAILING_METADATA,
+  GRPC_IOREQ_RECV_STATUS,
+  GRPC_IOREQ_RECV_STATUS_DETAILS,
+  GRPC_IOREQ_RECV_CLOSE,
+  GRPC_IOREQ_SEND_INITIAL_METADATA,
+  GRPC_IOREQ_SEND_MESSAGE,
+  GRPC_IOREQ_SEND_TRAILING_METADATA,
+  GRPC_IOREQ_SEND_STATUS,
+  GRPC_IOREQ_SEND_CLOSE,
+  GRPC_IOREQ_OP_COUNT
+} grpc_ioreq_op;
+
+typedef union {
+  grpc_metadata_array *recv_metadata;
+  grpc_byte_buffer **recv_message;
+  struct {
+    void (*set_value)(grpc_status_code status, void *user_data);
+    void *user_data;
+  } recv_status;
+  struct {
+    char **details;
+    size_t *details_capacity;
+  } recv_status_details;
+  struct {
+    size_t count;
+    const grpc_metadata *metadata;
+  } send_metadata;
+  grpc_byte_buffer *send_message;
+  struct {
+    grpc_status_code code;
+    const char *details;
+  } send_status;
+} grpc_ioreq_data;
+
+typedef struct {
+  grpc_ioreq_op op;
+  grpc_ioreq_data data;
+} grpc_ioreq;
+
+typedef void (*grpc_ioreq_completion_func)(grpc_call *call,
+                                           grpc_op_error status,
+                                           void *user_data);
+
+grpc_call *grpc_call_create(grpc_channel *channel, grpc_completion_queue *cq,
                             const void *server_transport_data);
 
+void grpc_call_set_completion_queue(grpc_call *call, grpc_completion_queue *cq);
+
 void grpc_call_internal_ref(grpc_call *call);
-void grpc_call_internal_unref(grpc_call *call);
+void grpc_call_internal_unref(grpc_call *call, int allow_immediate_deletion);
 
 /* Helpers for grpc_client, grpc_server filters to publish received data to
    the completion queue/surface layer */
 void grpc_call_recv_metadata(grpc_call_element *surface_element,
-                             grpc_call_op *op);
+                             grpc_mdelem *md);
-void grpc_call_recv_message(
+void grpc_call_recv_message(grpc_call_element *surface_element,
-    grpc_call_element *surface_element, grpc_byte_buffer *message,
+                            grpc_byte_buffer *message);
-    void (*on_finish)(void *user_data, grpc_op_error error), void *user_data);
+void grpc_call_read_closed(grpc_call_element *surface_element);
-void grpc_call_recv_finish(grpc_call_element *surface_element,
+void grpc_call_stream_closed(grpc_call_element *surface_element);
-                           int is_full_close);
 
 void grpc_call_execute_op(grpc_call *call, grpc_call_op *op);
+grpc_call_error grpc_call_start_ioreq_and_call_back(
+    grpc_call *call, const grpc_ioreq *reqs, size_t nreqs,
+    grpc_ioreq_completion_func on_complete, void *user_data);
 
-/* Called when it's known that the initial batch of metadata is complete on the
+/* Called when it's known that the initial batch of metadata is complete */
-   client side (must not be called on the server) */
+void grpc_call_initial_metadata_complete(grpc_call_element *surface_element);
-void grpc_call_client_initial_metadata_complete(
-    grpc_call_element *surface_element);
 
 void grpc_call_set_deadline(grpc_call_element *surface_element,
                             gpr_timespec deadline);
@@ -69,10 +118,4 @@ grpc_call_stack *grpc_call_get_call_stack(grpc_call *call);
 /* Given the top call_element, get the call object. */
 grpc_call *grpc_call_from_top_element(grpc_call_element *surface_element);
 
-/* Get the metadata buffer. */
-grpc_metadata_buffer *grpc_call_get_metadata_buffer(grpc_call *call);
-
-void grpc_call_add_mdelem(grpc_call *call, grpc_mdelem *mdelem,
-                          gpr_uint32 flags);
-
 #endif /* __GRPC_INTERNAL_SURFACE_CALL_H__ */

src/core/surface/call_details.c

@@ -0,0 +1,13 @@
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+
+#include <string.h>
+
+void grpc_call_details_init(grpc_call_details *cd) {
+  memset(cd, 0, sizeof(*cd));
+}
+
+void grpc_call_details_destroy(grpc_call_details *cd) {
+  gpr_free(cd->method);
+  gpr_free(cd->host);
+}

src/core/surface/channel.c

@@ -51,7 +51,10 @@ struct grpc_channel {
   grpc_mdstr *authority_string;
 };
 
-#define CHANNEL_STACK_FROM_CHANNEL(c) ((grpc_channel_stack *)((c) + 1))
+#define CHANNEL_STACK_FROM_CHANNEL(c) ((grpc_channel_stack *)((c)+1))
+#define CHANNEL_FROM_CHANNEL_STACK(channel_stack) (((grpc_channel *)(channel_stack)) - 1)
+#define CHANNEL_FROM_TOP_ELEM(top_elem) \
+  CHANNEL_FROM_CHANNEL_STACK(grpc_channel_stack_from_top_element(top_elem))
 
 grpc_channel *grpc_channel_create_from_filters(
     const grpc_channel_filter **filters, size_t num_filters,
@@ -60,8 +63,8 @@ grpc_channel *grpc_channel_create_from_filters(
       sizeof(grpc_channel) + grpc_channel_stack_size(filters, num_filters);
   grpc_channel *channel = gpr_malloc(size);
   channel->is_client = is_client;
-  /* decremented by grpc_channel_destroy */
+  /* decremented by grpc_channel_destroy, and grpc_client_channel_closed if is_client */
-  gpr_ref_init(&channel->refs, 1);
+  gpr_ref_init(&channel->refs, 1 + is_client);
   channel->metadata_context = mdctx;
   channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status");
   channel->grpc_message_string = grpc_mdstr_from_string(mdctx, "grpc-message");
@@ -74,37 +77,44 @@ grpc_channel *grpc_channel_create_from_filters(
 
 static void do_nothing(void *ignored, grpc_op_error error) {}
 
-grpc_call *grpc_channel_create_call_old(grpc_channel *channel,
+grpc_call *grpc_channel_create_call(grpc_channel *channel,
-                                        const char *method, const char *host,
+                                    grpc_completion_queue *cq,
-                                        gpr_timespec absolute_deadline) {
+                                    const char *method, const char *host,
+                                    gpr_timespec absolute_deadline) {
   grpc_call *call;
   grpc_mdelem *path_mdelem;
   grpc_mdelem *authority_mdelem;
+  grpc_call_op op;
 
   if (!channel->is_client) {
     gpr_log(GPR_ERROR, "Cannot create a call on the server.");
     return NULL;
   }
 
-  call = grpc_call_create(channel, NULL);
+  call = grpc_call_create(channel, cq, NULL);
 
   /* Add :path and :authority headers. */
   /* TODO(klempner): Consider optimizing this by stashing mdelems for common
      values of method and host. */
-  grpc_mdstr_ref(channel->path_string);
   path_mdelem = grpc_mdelem_from_metadata_strings(
-      channel->metadata_context, channel->path_string,
+      channel->metadata_context, grpc_mdstr_ref(channel->path_string),
       grpc_mdstr_from_string(channel->metadata_context, method));
-  grpc_call_add_mdelem(call, path_mdelem, 0);
+  op.type = GRPC_SEND_METADATA;
+  op.dir = GRPC_CALL_DOWN;
+  op.flags = 0;
+  op.data.metadata = path_mdelem;
+  op.done_cb = do_nothing;
+  op.user_data = NULL;
+  grpc_call_execute_op(call, &op);
 
   grpc_mdstr_ref(channel->authority_string);
   authority_mdelem = grpc_mdelem_from_metadata_strings(
       channel->metadata_context, channel->authority_string,
       grpc_mdstr_from_string(channel->metadata_context, host));
-  grpc_call_add_mdelem(call, authority_mdelem, 0);
+  op.data.metadata = authority_mdelem;
+  grpc_call_execute_op(call, &op);
 
   if (0 != gpr_time_cmp(absolute_deadline, gpr_inf_future)) {
-    grpc_call_op op;
     op.type = GRPC_SEND_DEADLINE;
     op.dir = GRPC_CALL_DOWN;
     op.flags = 0;
@@ -117,6 +127,13 @@ grpc_call *grpc_channel_create_call_old(grpc_channel *channel,
   return call;
 }
 
+grpc_call *grpc_channel_create_call_old(grpc_channel *channel,
+                                        const char *method, const char *host,
+                                        gpr_timespec absolute_deadline) {
+  return grpc_channel_create_call(channel, NULL, method, host,
+                                  absolute_deadline);
+}
+
 void grpc_channel_internal_ref(grpc_channel *channel) {
   gpr_ref(&channel->refs);
 }
@@ -152,6 +169,10 @@ void grpc_channel_destroy(grpc_channel *channel) {
   grpc_channel_internal_unref(channel);
 }
 
+void grpc_client_channel_closed(grpc_channel_element *elem) {
+  grpc_channel_internal_unref(CHANNEL_FROM_TOP_ELEM(elem));
+}
+
 grpc_channel_stack *grpc_channel_get_channel_stack(grpc_channel *channel) {
   return CHANNEL_STACK_FROM_CHANNEL(channel);
 }

src/core/surface/channel.h

@@ -45,6 +45,8 @@ grpc_mdctx *grpc_channel_get_metadata_context(grpc_channel *channel);
 grpc_mdstr *grpc_channel_get_status_string(grpc_channel *channel);
 grpc_mdstr *grpc_channel_get_message_string(grpc_channel *channel);
 
+void grpc_client_channel_closed(grpc_channel_element *elem);
+
 void grpc_channel_internal_ref(grpc_channel *channel);
 void grpc_channel_internal_unref(grpc_channel *channel);
 

src/core/surface/client.c

@@ -34,6 +34,7 @@
 #include "src/core/surface/client.h"
 
 #include "src/core/surface/call.h"
+#include "src/core/surface/channel.h"
 #include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
@@ -56,23 +57,23 @@ static void call_op(grpc_call_element *elem, grpc_call_element *from_elem,
       grpc_call_next_op(elem, op);
       break;
     case GRPC_RECV_METADATA:
-      grpc_call_recv_metadata(elem, op);
+      grpc_call_recv_metadata(elem, op->data.metadata);
       break;
     case GRPC_RECV_DEADLINE:
       gpr_log(GPR_ERROR, "Deadline received by client (ignored)");
       break;
     case GRPC_RECV_MESSAGE:
-      grpc_call_recv_message(elem, op->data.message, op->done_cb,
+      grpc_call_recv_message(elem, op->data.message);
-                             op->user_data);
+      op->done_cb(op->user_data, GRPC_OP_OK);
       break;
     case GRPC_RECV_HALF_CLOSE:
-      grpc_call_recv_finish(elem, 0);
+      grpc_call_read_closed(elem);
       break;
     case GRPC_RECV_FINISH:
-      grpc_call_recv_finish(elem, 1);
+      grpc_call_stream_closed(elem);
       break;
     case GRPC_RECV_END_OF_INITIAL_METADATA:
-      grpc_call_client_initial_metadata_complete(elem);
+      grpc_call_initial_metadata_complete(elem);
       break;
     default:
       GPR_ASSERT(op->dir == GRPC_CALL_DOWN);
@@ -87,7 +88,7 @@ static void channel_op(grpc_channel_element *elem,
       gpr_log(GPR_ERROR, "Client cannot accept new calls");
       break;
     case GRPC_TRANSPORT_CLOSED:
-      gpr_log(GPR_ERROR, "Transport closed");
+      grpc_client_channel_closed(elem);
       break;
     case GRPC_TRANSPORT_GOAWAY:
       gpr_slice_unref(op->data.goaway.message);

src/core/surface/completion_queue.c

@@ -173,18 +173,6 @@ void grpc_cq_end_read(grpc_completion_queue *cc, void *tag, grpc_call *call,
   gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
 }
 
-void grpc_cq_end_invoke_accepted(grpc_completion_queue *cc, void *tag,
-                                 grpc_call *call,
-                                 grpc_event_finish_func on_finish,
-                                 void *user_data, grpc_op_error error) {
-  event *ev;
-  gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
-  ev = add_locked(cc, GRPC_INVOKE_ACCEPTED, tag, call, on_finish, user_data);
-  ev->base.data.invoke_accepted = error;
-  end_op_locked(cc, GRPC_INVOKE_ACCEPTED);
-  gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
-}
-
 void grpc_cq_end_write_accepted(grpc_completion_queue *cc, void *tag,
                                 grpc_call *call,
                                 grpc_event_finish_func on_finish,
@@ -197,6 +185,28 @@ void grpc_cq_end_write_accepted(grpc_completion_queue *cc, void *tag,
   gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
 }
 
+void grpc_cq_end_op_complete(grpc_completion_queue *cc, void *tag,
+                             grpc_call *call, grpc_event_finish_func on_finish,
+                             void *user_data, grpc_op_error error) {
+  event *ev;
+  gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
+  ev = add_locked(cc, GRPC_OP_COMPLETE, tag, call, on_finish, user_data);
+  ev->base.data.write_accepted = error;
+  end_op_locked(cc, GRPC_OP_COMPLETE);
+  gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
+}
+
+void grpc_cq_end_op(grpc_completion_queue *cc, void *tag, grpc_call *call,
+                    grpc_event_finish_func on_finish, void *user_data,
+                    grpc_op_error error) {
+  event *ev;
+  gpr_mu_lock(GRPC_POLLSET_MU(&cc->pollset));
+  ev = add_locked(cc, GRPC_OP_COMPLETE, tag, call, on_finish, user_data);
+  ev->base.data.write_accepted = error;
+  end_op_locked(cc, GRPC_OP_COMPLETE);
+  gpr_mu_unlock(GRPC_POLLSET_MU(&cc->pollset));
+}
+
 void grpc_cq_end_finish_accepted(grpc_completion_queue *cc, void *tag,
                                  grpc_call *call,
                                  grpc_event_finish_func on_finish,
@@ -389,7 +399,7 @@ void grpc_event_finish(grpc_event *base) {
   event *ev = (event *)base;
   ev->on_finish(ev->on_finish_user_data, GRPC_OP_OK);
   if (ev->base.call) {
-    grpc_call_internal_unref(ev->base.call);
+    grpc_call_internal_unref(ev->base.call, 1);
   }
   gpr_free(ev);
 }

src/core/surface/completion_queue.h

@@ -78,6 +78,10 @@ void grpc_cq_end_finish_accepted(grpc_completion_queue *cc, void *tag,
                                  grpc_call *call,
                                  grpc_event_finish_func on_finish,
                                  void *user_data, grpc_op_error error);
+/* Queue a GRPC_OP_COMPLETED operation */
+void grpc_cq_end_op_complete(grpc_completion_queue *cc, void *tag,
+                             grpc_call *call, grpc_event_finish_func on_finish,
+                             void *user_data, grpc_op_error error);
 /* Queue a GRPC_CLIENT_METADATA_READ operation */
 void grpc_cq_end_client_metadata_read(grpc_completion_queue *cc, void *tag,
                                       grpc_call *call,
@@ -97,6 +101,10 @@ void grpc_cq_end_new_rpc(grpc_completion_queue *cc, void *tag, grpc_call *call,
                          gpr_timespec deadline, size_t metadata_count,
                          grpc_metadata *metadata_elements);
 
+void grpc_cq_end_op(grpc_completion_queue *cc, void *tag, grpc_call *call,
+                    grpc_event_finish_func on_finish, void *user_data,
+                    grpc_op_error error);
+
 void grpc_cq_end_server_shutdown(grpc_completion_queue *cc, void *tag);
 
 /* disable polling for some tests */

src/core/surface/event_string.c

@@ -87,10 +87,10 @@ char *grpc_event_string(grpc_event *ev) {
         gpr_strvec_add(&buf, gpr_strdup(" end-of-stream"));
       }
       break;
-    case GRPC_INVOKE_ACCEPTED:
+    case GRPC_OP_COMPLETE:
-      gpr_strvec_add(&buf, gpr_strdup("INVOKE_ACCEPTED: "));
+      gpr_strvec_add(&buf, gpr_strdup("OP_COMPLETE: "));
       addhdr(&buf, ev);
-      adderr(&buf, ev->data.invoke_accepted);
+      adderr(&buf, ev->data.op_complete);
       break;
     case GRPC_WRITE_ACCEPTED:
       gpr_strvec_add(&buf, gpr_strdup("WRITE_ACCEPTED: "));

src/core/surface/lame_client.c

@@ -50,26 +50,16 @@ typedef struct {
   grpc_mdelem *message;
 } channel_data;
 
-static void do_nothing(void *data, grpc_op_error error) {}
-
 static void call_op(grpc_call_element *elem, grpc_call_element *from_elem,
                     grpc_call_op *op) {
   channel_data *channeld = elem->channel_data;
   GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
 
   switch (op->type) {
-    case GRPC_SEND_START: {
+    case GRPC_SEND_START:
-      grpc_call_op set_status_op;
+      grpc_call_recv_metadata(elem, grpc_mdelem_ref(channeld->message));
-      grpc_mdelem_ref(channeld->message);
+      grpc_call_stream_closed(elem);
-      memset(&set_status_op, 0, sizeof(grpc_call_op));
-      set_status_op.dir = GRPC_CALL_UP;
-      set_status_op.type = GRPC_RECV_METADATA;
-      set_status_op.done_cb = do_nothing;
-      set_status_op.data.metadata = channeld->message;
-      grpc_call_recv_metadata(elem, &set_status_op);
-      grpc_call_recv_finish(elem, 1);
       break;
-    }
     case GRPC_SEND_METADATA:
       grpc_mdelem_unref(op->data.metadata);
       break;
@@ -86,6 +76,9 @@ static void channel_op(grpc_channel_element *elem,
     case GRPC_CHANNEL_GOAWAY:
       gpr_slice_unref(op->data.goaway.message);
       break;
+    case GRPC_CHANNEL_DISCONNECT:
+      grpc_client_channel_closed(elem);
+      break;
     default:
       break;
   }

src/core/surface/metadata_array.c

@@ -0,0 +1,12 @@
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+
+#include <string.h>
+
+void grpc_metadata_array_init(grpc_metadata_array *array) {
+  memset(array, 0, sizeof(*array));
+}
+
+void grpc_metadata_array_destroy(grpc_metadata_array *array) {
+  gpr_free(array->metadata);
+}

src/core/surface/server.c

@@ -44,6 +44,7 @@
 #include "src/core/surface/call.h"
 #include "src/core/surface/channel.h"
 #include "src/core/surface/completion_queue.h"
+#include "src/core/transport/metadata.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
@@ -63,11 +64,27 @@ typedef struct channel_data channel_data;
 struct channel_data {
   grpc_server *server;
   grpc_channel *channel;
+  grpc_mdstr *path_key;
+  grpc_mdstr *authority_key;
   /* linked list of all channels on a server */
   channel_data *next;
   channel_data *prev;
 };
 
+typedef void (*new_call_cb)(grpc_server *server, grpc_completion_queue *cq,
+                            grpc_call **call, grpc_call_details *details,
+                            grpc_metadata_array *initial_metadata,
+                            call_data *calld, void *user_data);
+
+typedef struct {
+  void *user_data;
+  grpc_completion_queue *cq;
+  grpc_call **call;
+  grpc_call_details *details;
+  grpc_metadata_array *initial_metadata;
+  new_call_cb cb;
+} requested_call;
+
 struct grpc_server {
   size_t channel_filter_count;
   const grpc_channel_filter **channel_filters;
@@ -76,9 +93,9 @@ struct grpc_server {
 
   gpr_mu mu;
 
-  void **tags;
+  requested_call *requested_calls;
-  size_t ntags;
+  size_t requested_call_count;
-  size_t tag_cap;
+  size_t requested_call_capacity;
 
   gpr_uint8 shutdown;
   gpr_uint8 have_shutdown_tag;
@@ -107,11 +124,20 @@ typedef enum {
   ZOMBIED
 } call_state;
 
+typedef struct legacy_data {
+  grpc_metadata_array *initial_metadata;
+} legacy_data;
+
 struct call_data {
   grpc_call *call;
 
   call_state state;
   gpr_timespec deadline;
+  grpc_mdstr *path;
+  grpc_mdstr *host;
+
+  legacy_data *legacy;
+  grpc_call_details *details;
 
   gpr_uint8 included[CALL_LIST_COUNT];
   call_link links[CALL_LIST_COUNT];
@@ -179,7 +205,7 @@ static void server_unref(grpc_server *server) {
     grpc_channel_args_destroy(server->channel_args);
     gpr_mu_destroy(&server->mu);
     gpr_free(server->channel_filters);
-    gpr_free(server->tags);
+    gpr_free(server->requested_calls);
     gpr_free(server);
   }
 }
@@ -210,62 +236,37 @@ static void destroy_channel(channel_data *chand) {
   grpc_iomgr_add_callback(finish_destroy_channel, chand);
 }
 
-static void queue_new_rpc(grpc_server *server, call_data *calld, void *tag) {
-  grpc_call *call = calld->call;
-  grpc_metadata_buffer *mdbuf = grpc_call_get_metadata_buffer(call);
-  size_t count = grpc_metadata_buffer_count(mdbuf);
-  grpc_metadata *elements = grpc_metadata_buffer_extract_elements(mdbuf);
-  const char *host = NULL;
-  const char *method = NULL;
-  size_t i;
-
-  for (i = 0; i < count; i++) {
-    if (0 == strcmp(elements[i].key, ":authority")) {
-      host = elements[i].value;
-    } else if (0 == strcmp(elements[i].key, ":path")) {
-      method = elements[i].value;
-    }
-  }
-
-  grpc_call_internal_ref(call);
-  grpc_cq_end_new_rpc(server->cq, tag, call,
-                      grpc_metadata_buffer_cleanup_elements, elements, method,
-                      host, calld->deadline, count, elements);
-}
-
 static void start_new_rpc(grpc_call_element *elem) {
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   grpc_server *server = chand->server;
 
   gpr_mu_lock(&server->mu);
-  if (server->ntags) {
+  if (server->requested_call_count > 0) {
+    requested_call rc = server->requested_calls[--server->requested_call_count];
     calld->state = ACTIVATED;
-    queue_new_rpc(server, calld, server->tags[--server->ntags]);
+    gpr_mu_unlock(&server->mu);
+    rc.cb(server, rc.cq, rc.call, rc.details, rc.initial_metadata, calld,
+          rc.user_data);
   } else {
     calld->state = PENDING;
     call_list_join(server, calld, PENDING_START);
+    gpr_mu_unlock(&server->mu);
   }
-  gpr_mu_unlock(&server->mu);
 }
 
 static void kill_zombie(void *elem, int success) {
   grpc_call_destroy(grpc_call_from_top_element(elem));
 }
 
-static void finish_rpc(grpc_call_element *elem, int is_full_close) {
+static void stream_closed(grpc_call_element *elem) {
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
   gpr_mu_lock(&chand->server->mu);
   switch (calld->state) {
     case ACTIVATED:
-      grpc_call_recv_finish(elem, is_full_close);
       break;
     case PENDING:
-      if (!is_full_close) {
-        grpc_call_recv_finish(elem, is_full_close);
-        break;
-      }
       call_list_remove(chand->server, calld, PENDING_START);
     /* fallthrough intended */
     case NOT_STARTED:
@@ -276,27 +277,60 @@ static void finish_rpc(grpc_call_element *elem, int is_full_close) {
       break;
   }
   gpr_mu_unlock(&chand->server->mu);
+  grpc_call_stream_closed(elem);
+}
+
+static void read_closed(grpc_call_element *elem) {
+  call_data *calld = elem->call_data;
+  channel_data *chand = elem->channel_data;
+  gpr_mu_lock(&chand->server->mu);
+  switch (calld->state) {
+    case ACTIVATED:
+    case PENDING:
+      grpc_call_read_closed(elem);
+      break;
+    case NOT_STARTED:
+      calld->state = ZOMBIED;
+      grpc_iomgr_add_callback(kill_zombie, elem);
+      break;
+    case ZOMBIED:
+      break;
+  }
+  gpr_mu_unlock(&chand->server->mu);
 }
 
 static void call_op(grpc_call_element *elem, grpc_call_element *from_elemn,
                     grpc_call_op *op) {
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  grpc_mdelem *md;
   GRPC_CALL_LOG_OP(GPR_INFO, elem, op);
   switch (op->type) {
     case GRPC_RECV_METADATA:
-      grpc_call_recv_metadata(elem, op);
+      md = op->data.metadata;
+      if (md->key == chand->path_key) {
+        calld->path = grpc_mdstr_ref(md->value);
+        grpc_mdelem_unref(md);
+      } else if (md->key == chand->authority_key) {
+        calld->host = grpc_mdstr_ref(md->value);
+        grpc_mdelem_unref(md);
+      } else {
+        grpc_call_recv_metadata(elem, md);
+      }
       break;
     case GRPC_RECV_END_OF_INITIAL_METADATA:
       start_new_rpc(elem);
+      grpc_call_initial_metadata_complete(elem);
       break;
     case GRPC_RECV_MESSAGE:
-      grpc_call_recv_message(elem, op->data.message, op->done_cb,
+      grpc_call_recv_message(elem, op->data.message);
-                             op->user_data);
+      op->done_cb(op->user_data, GRPC_OP_OK);
       break;
     case GRPC_RECV_HALF_CLOSE:
-      finish_rpc(elem, 0);
+      read_closed(elem);
       break;
     case GRPC_RECV_FINISH:
-      finish_rpc(elem, 1);
+      stream_closed(elem);
       break;
     case GRPC_RECV_DEADLINE:
       grpc_call_set_deadline(elem, op->data.deadline);
@@ -312,21 +346,22 @@ static void call_op(grpc_call_element *elem, grpc_call_element *from_elemn,
 static void channel_op(grpc_channel_element *elem,
                        grpc_channel_element *from_elem, grpc_channel_op *op) {
   channel_data *chand = elem->channel_data;
+  grpc_server *server = chand->server;
 
   switch (op->type) {
     case GRPC_ACCEPT_CALL:
       /* create a call */
-      grpc_call_create(chand->channel,
+      grpc_call_create(chand->channel, NULL,
                        op->data.accept_call.transport_server_data);
       break;
     case GRPC_TRANSPORT_CLOSED:
       /* if the transport is closed for a server channel, we destroy the
          channel */
-      gpr_mu_lock(&chand->server->mu);
+      gpr_mu_lock(&server->mu);
-      server_ref(chand->server);
+      server_ref(server);
       destroy_channel(chand);
-      gpr_mu_unlock(&chand->server->mu);
+      gpr_mu_unlock(&server->mu);
-      server_unref(chand->server);
+      server_unref(server);
       break;
     case GRPC_TRANSPORT_GOAWAY:
       gpr_slice_unref(op->data.goaway.message);
@@ -371,6 +406,7 @@ static void init_call_elem(grpc_call_element *elem,
 
 static void destroy_call_elem(grpc_call_element *elem) {
   channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
   int i;
 
   gpr_mu_lock(&chand->server->mu);
@@ -383,6 +419,19 @@ static void destroy_call_elem(grpc_call_element *elem) {
   }
   gpr_mu_unlock(&chand->server->mu);
 
+  if (calld->host) {
+    grpc_mdstr_unref(calld->host);
+  }
+  if (calld->path) {
+    grpc_mdstr_unref(calld->path);
+  }
+
+  if (calld->legacy) {
+    gpr_free(calld->legacy->initial_metadata->metadata);
+    gpr_free(calld->legacy->initial_metadata);
+    gpr_free(calld->legacy);
+  }
+
   server_unref(chand->server);
 }
 
@@ -395,6 +444,8 @@ static void init_channel_elem(grpc_channel_element *elem,
   GPR_ASSERT(!is_last);
   chand->server = NULL;
   chand->channel = NULL;
+  chand->path_key = grpc_mdstr_from_string(metadata_context, ":path");
+  chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority");
   chand->next = chand->prev = chand;
 }
 
@@ -406,6 +457,8 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
     chand->prev->next = chand->next;
     chand->next = chand->prev = chand;
     gpr_mu_unlock(&chand->server->mu);
+    grpc_mdstr_unref(chand->path_key);
+    grpc_mdstr_unref(chand->authority_key);
     server_unref(chand->server);
   }
 }
@@ -413,17 +466,8 @@ static void destroy_channel_elem(grpc_channel_element *elem) {
 static const grpc_channel_filter server_surface_filter = {
     call_op,           channel_op,           sizeof(call_data),
     init_call_elem,    destroy_call_elem,    sizeof(channel_data),
-    init_channel_elem, destroy_channel_elem, "server", };
+    init_channel_elem, destroy_channel_elem, "server",
-
+};
-static void early_terminate_requested_calls(grpc_completion_queue *cq,
-                                            void **tags, size_t ntags) {
-  size_t i;
-
-  for (i = 0; i < ntags; i++) {
-    grpc_cq_end_new_rpc(cq, tags[i], NULL, do_nothing, NULL, NULL, NULL,
-                        gpr_inf_past, 0, NULL);
-  }
-}
 
 grpc_server *grpc_server_create_from_filters(grpc_completion_queue *cq,
                                              grpc_channel_filter **filters,
@@ -517,8 +561,8 @@ grpc_transport_setup_result grpc_server_setup_transport(
 void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
                        void *shutdown_tag) {
   listener *l;
-  void **tags;
+  requested_call *requested_calls;
-  size_t ntags;
+  size_t requested_call_count;
   channel_data **channels;
   channel_data *c;
   size_t nchannels;
@@ -547,10 +591,10 @@ void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
     i++;
   }
 
-  tags = server->tags;
+  requested_calls = server->requested_calls;
-  ntags = server->ntags;
+  requested_call_count = server->requested_call_count;
-  server->tags = NULL;
+  server->requested_calls = NULL;
-  server->ntags = 0;
+  server->requested_call_count = 0;
 
   server->shutdown = 1;
   server->have_shutdown_tag = have_shutdown_tag;
@@ -579,8 +623,13 @@ void shutdown_internal(grpc_server *server, gpr_uint8 have_shutdown_tag,
   gpr_free(channels);
 
   /* terminate all the requested calls */
-  early_terminate_requested_calls(server->cq, tags, ntags);
+  for (i = 0; i < requested_call_count; i++) {
-  gpr_free(tags);
+    requested_calls[i].cb(server, requested_calls[i].cq,
+                          requested_calls[i].call, requested_calls[i].details,
+                          requested_calls[i].initial_metadata, NULL,
+                          requested_calls[i].user_data);
+  }
+  gpr_free(requested_calls);
 
   /* Shutdown listeners */
   for (l = server->listeners; l; l = l->next) {
@@ -625,36 +674,158 @@ void grpc_server_add_listener(grpc_server *server, void *arg,
   server->listeners = l;
 }
 
-grpc_call_error grpc_server_request_call_old(grpc_server *server,
+static grpc_call_error queue_call_request(grpc_server *server,
-                                             void *tag_new) {
+                                          grpc_completion_queue *cq,
+                                          grpc_call **call,
+                                          grpc_call_details *details,
+                                          grpc_metadata_array *initial_metadata,
+                                          new_call_cb cb, void *user_data) {
   call_data *calld;
-
+  requested_call *rc;
-  grpc_cq_begin_op(server->cq, NULL, GRPC_SERVER_RPC_NEW);
-
   gpr_mu_lock(&server->mu);
-
   if (server->shutdown) {
     gpr_mu_unlock(&server->mu);
-    early_terminate_requested_calls(server->cq, &tag_new, 1);
+    cb(server, cq, call, details, initial_metadata, NULL, user_data);
     return GRPC_CALL_OK;
   }
-
   calld = call_list_remove_head(server, PENDING_START);
   if (calld) {
     GPR_ASSERT(calld->state == PENDING);
     calld->state = ACTIVATED;
-    queue_new_rpc(server, calld, tag_new);
+    gpr_mu_unlock(&server->mu);
+    cb(server, cq, call, details, initial_metadata, calld, user_data);
+    return GRPC_CALL_OK;
   } else {
-    if (server->tag_cap == server->ntags) {
+    if (server->requested_call_count == server->requested_call_capacity) {
-      server->tag_cap = GPR_MAX(3 * server->tag_cap / 2, server->tag_cap + 1);
+      server->requested_call_capacity =
-      server->tags =
+          GPR_MAX(server->requested_call_capacity + 8,
-          gpr_realloc(server->tags, sizeof(void *) * server->tag_cap);
+                  server->requested_call_capacity * 2);
+      server->requested_calls =
+          gpr_realloc(server->requested_calls,
+                      sizeof(requested_call) * server->requested_call_capacity);
     }
-    server->tags[server->ntags++] = tag_new;
+    rc = &server->requested_calls[server->requested_call_count++];
+    rc->cb = cb;
+    rc->cq = cq;
+    rc->call = call;
+    rc->details = details;
+    rc->user_data = user_data;
+    rc->initial_metadata = initial_metadata;
+    gpr_mu_unlock(&server->mu);
+    return GRPC_CALL_OK;
   }
-  gpr_mu_unlock(&server->mu);
+}
+
+static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) {
+  gpr_slice slice = value->slice;
+  size_t len = GPR_SLICE_LENGTH(slice);
 
-  return GRPC_CALL_OK;
+  if (len + 1 > *capacity) {
+    *capacity = GPR_MAX(len + 1, *capacity * 2);
+    *dest = gpr_realloc(*dest, *capacity);
+  }
+  memcpy(*dest, grpc_mdstr_as_c_string(value), len + 1);
+}
+
+static void publish_request(grpc_call *call, grpc_op_error status, void *tag) {
+  grpc_call_element *elem =
+      grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
+  call_data *calld = elem->call_data;
+  channel_data *chand = elem->channel_data;
+  grpc_server *server = chand->server;
+
+  if (status == GRPC_OP_OK) {
+    cpstr(&calld->details->host, &calld->details->host_capacity, calld->host);
+    cpstr(&calld->details->method, &calld->details->method_capacity,
+          calld->path);
+    calld->details->deadline = calld->deadline;
+    grpc_cq_end_op_complete(server->cq, tag, call, do_nothing, NULL,
+                            GRPC_OP_OK);
+  } else {
+    abort();
+  }
+}
+
+static void begin_request(grpc_server *server, grpc_completion_queue *cq,
+                          grpc_call **call, grpc_call_details *details,
+                          grpc_metadata_array *initial_metadata,
+                          call_data *calld, void *tag) {
+  grpc_ioreq req;
+  if (!calld) {
+    *call = NULL;
+    initial_metadata->count = 0;
+    grpc_cq_end_op_complete(cq, tag, NULL, do_nothing, NULL, GRPC_OP_ERROR);
+    return;
+  }
+  calld->details = details;
+  grpc_call_set_completion_queue(calld->call, cq);
+  *call = calld->call;
+  req.op = GRPC_IOREQ_RECV_INITIAL_METADATA;
+  req.data.recv_metadata = initial_metadata;
+  grpc_call_internal_ref(calld->call);
+  grpc_call_start_ioreq_and_call_back(calld->call, &req, 1, publish_request,
+                                      tag);
+}
+
+grpc_call_error grpc_server_request_call(grpc_server *server, grpc_call **call,
+                                         grpc_call_details *details,
+                                         grpc_metadata_array *initial_metadata,
+                                         grpc_completion_queue *cq, void *tag) {
+  grpc_cq_begin_op(cq, NULL, GRPC_OP_COMPLETE);
+  return queue_call_request(server, cq, call, details, initial_metadata,
+                            begin_request, tag);
+}
+
+static void publish_legacy_request(grpc_call *call, grpc_op_error status,
+                                   void *tag) {
+  grpc_call_element *elem =
+      grpc_call_stack_element(grpc_call_get_call_stack(call), 0);
+  call_data *calld = elem->call_data;
+  channel_data *chand = elem->channel_data;
+  grpc_server *server = chand->server;
+
+  if (status == GRPC_OP_OK) {
+    grpc_cq_end_new_rpc(server->cq, tag, call, do_nothing, NULL,
+                        grpc_mdstr_as_c_string(calld->path),
+                        grpc_mdstr_as_c_string(calld->host), calld->deadline,
+                        calld->legacy->initial_metadata->count,
+                        calld->legacy->initial_metadata->metadata);
+  } else {
+    abort();
+  }
+}
+
+static void begin_legacy_request(grpc_server *server, grpc_completion_queue *cq,
+                                 grpc_call **call, grpc_call_details *details,
+                                 grpc_metadata_array *initial_metadata,
+                                 call_data *calld, void *tag) {
+  grpc_ioreq req;
+  GPR_ASSERT(call == NULL);
+  GPR_ASSERT(details == NULL);
+  if (!calld) {
+    gpr_free(initial_metadata);
+    grpc_cq_end_new_rpc(cq, tag, NULL, do_nothing, NULL, NULL, NULL,
+                        gpr_inf_past, 0, NULL);
+    return;
+  }
+  req.op = GRPC_IOREQ_RECV_INITIAL_METADATA;
+  req.data.recv_metadata = initial_metadata;
+  calld->legacy = gpr_malloc(sizeof(legacy_data));
+  memset(calld->legacy, 0, sizeof(legacy_data));
+  calld->legacy->initial_metadata = initial_metadata;
+  grpc_call_internal_ref(calld->call);
+  grpc_call_start_ioreq_and_call_back(calld->call, &req, 1,
+                                      publish_legacy_request, tag);
+}
+
+grpc_call_error grpc_server_request_call_old(grpc_server *server,
+                                             void *tag_new) {
+  grpc_metadata_array *client_metadata =
+      gpr_malloc(sizeof(grpc_metadata_array));
+  memset(client_metadata, 0, sizeof(*client_metadata));
+  grpc_cq_begin_op(server->cq, NULL, GRPC_SERVER_RPC_NEW);
+  return queue_call_request(server, server->cq, NULL, NULL, client_metadata,
+                            begin_legacy_request, tag_new);
 }
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server) {

src/core/transport/chttp2/stream_encoder.c

@@ -432,7 +432,7 @@ static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem,
 
 static void deadline_enc(grpc_chttp2_hpack_compressor *c, gpr_timespec deadline,
                          framer_state *st) {
-  char timeout_str[32];
+  char timeout_str[GRPC_CHTTP2_TIMEOUT_ENCODE_MIN_BUFSIZE];
   grpc_chttp2_encode_timeout(gpr_time_sub(deadline, gpr_now()), timeout_str);
   hpack_enc(c, grpc_mdelem_from_metadata_strings(
                    c->mdctx, grpc_mdstr_ref(c->timeout_key_str),

src/core/transport/chttp2_transport.c

@@ -184,7 +184,6 @@ struct transport {
   gpr_uint8 is_client;
 
   gpr_mu mu;
-  gpr_cv cv;
 
   /* basic state management - what are we doing at the moment? */
   gpr_uint8 reading;
@@ -328,6 +327,9 @@ static void maybe_start_some_streams(transport *t);
 
 static void become_skip_parser(transport *t);
 
+static void recv_data(void *tp, gpr_slice *slices, size_t nslices,
+                      grpc_endpoint_cb_status error);
+
 /*
  * CONSTRUCTION/DESTRUCTION/REFCOUNTING
  */
@@ -382,8 +384,8 @@ static void ref_transport(transport *t) { gpr_ref(&t->refs); }
 
 static void init_transport(transport *t, grpc_transport_setup_callback setup,
                            void *arg, const grpc_channel_args *channel_args,
-                           grpc_endpoint *ep, grpc_mdctx *mdctx,
+                           grpc_endpoint *ep, gpr_slice *slices, size_t nslices,
-                           int is_client) {
+                           grpc_mdctx *mdctx, int is_client) {
   size_t i;
   int j;
   grpc_transport_setup_result sr;
@@ -395,7 +397,6 @@ static void init_transport(transport *t, grpc_transport_setup_callback setup,
   /* one ref is for destroy, the other for when ep becomes NULL */
   gpr_ref_init(&t->refs, 2);
   gpr_mu_init(&t->mu);
-  gpr_cv_init(&t->cv);
   t->metadata_context = mdctx;
   t->str_grpc_timeout =
       grpc_mdstr_from_string(t->metadata_context, "grpc-timeout");
@@ -422,6 +423,7 @@ static void init_transport(transport *t, grpc_transport_setup_callback setup,
   gpr_slice_buffer_init(&t->outbuf);
   gpr_slice_buffer_init(&t->qbuf);
   grpc_sopb_init(&t->nuke_later_sopb);
+  grpc_chttp2_hpack_parser_init(&t->hpack_parser, t->metadata_context);
   if (is_client) {
     gpr_slice_buffer_add(&t->qbuf,
                          gpr_slice_from_copied_string(CLIENT_CONNECT_STRING));
@@ -476,14 +478,15 @@ static void init_transport(transport *t, grpc_transport_setup_callback setup,
   ref_transport(t);
   gpr_mu_unlock(&t->mu);
 
+  ref_transport(t);
+  recv_data(t, slices, nslices, GRPC_ENDPOINT_CB_OK);
+
   sr = setup(arg, &t->base, t->metadata_context);
 
   lock(t);
   t->cb = sr.callbacks;
   t->cb_user_data = sr.user_data;
-  grpc_chttp2_hpack_parser_init(&t->hpack_parser, t->metadata_context);
   t->calling_back = 0;
-  gpr_cv_broadcast(&t->cv);
   unlock(t);
   unref_transport(t);
 }
@@ -492,9 +495,6 @@ static void destroy_transport(grpc_transport *gt) {
   transport *t = (transport *)gt;
 
   gpr_mu_lock(&t->mu);
-  while (t->calling_back) {
-    gpr_cv_wait(&t->cv, &t->mu, gpr_inf_future);
-  }
   t->cb = NULL;
   gpr_mu_unlock(&t->mu);
 
@@ -573,13 +573,6 @@ static void destroy_stream(grpc_transport *gt, grpc_stream *gs) {
 
   gpr_mu_lock(&t->mu);
 
-  /* await pending callbacks
-     TODO(ctiller): this could be optimized to check if this stream is getting
-     callbacks */
-  while (t->calling_back) {
-    gpr_cv_wait(&t->cv, &t->mu, gpr_inf_future);
-  }
-
   /* stop parsing if we're currently parsing this stream */
   if (t->deframe_state == DTS_FRAME && t->incoming_stream_id == s->id &&
       s->id != 0) {
@@ -591,7 +584,6 @@ static void destroy_stream(grpc_transport *gt, grpc_stream *gs) {
   }
   remove_from_stream_map(t, s);
 
-  gpr_cv_broadcast(&t->cv);
   gpr_mu_unlock(&t->mu);
 
   grpc_sopb_destroy(&s->outgoing_sopb);
@@ -761,7 +753,6 @@ static void unlock(transport *t) {
   if (perform_callbacks || call_closed || num_goaways) {
     lock(t);
     t->calling_back = 0;
-    gpr_cv_broadcast(&t->cv);
     unlock(t);
     unref_transport(t);
   }
@@ -892,7 +883,6 @@ static void finish_write_common(transport *t, int success) {
   if (!t->reading) {
     grpc_endpoint_destroy(t->ep);
     t->ep = NULL;
-    gpr_cv_broadcast(&t->cv);
     unref_transport(t); /* safe because we'll still have the ref for write */
   }
   unlock(t);
@@ -957,7 +947,7 @@ static void send_batch(grpc_transport *gt, grpc_stream *gs, grpc_stream_op *ops,
       stream_list_join(t, s, WRITABLE);
     }
   } else {
-    grpc_stream_ops_unref_owned_objects(ops, ops_count);
+    grpc_sopb_append(&t->nuke_later_sopb, ops, ops_count);
   }
   if (is_last && s->outgoing_sopb.nops == 0 && s->read_closed) {
     stream_list_join(t, s, PENDING_CALLBACKS);
@@ -1673,7 +1663,6 @@ static void recv_data(void *tp, gpr_slice *slices, size_t nslices,
       if (!t->writing && t->ep) {
         grpc_endpoint_destroy(t->ep);
         t->ep = NULL;
-        gpr_cv_broadcast(&t->cv);
         unref_transport(t); /* safe as we still have a ref for read */
       }
       unlock(t);
@@ -1769,7 +1758,6 @@ void grpc_create_chttp2_transport(grpc_transport_setup_callback setup,
                                   size_t nslices, grpc_mdctx *mdctx,
                                   int is_client) {
   transport *t = gpr_malloc(sizeof(transport));
-  init_transport(t, setup, arg, channel_args, ep, mdctx, is_client);
+  init_transport(t, setup, arg, channel_args, ep, slices, nslices, mdctx,
-  ref_transport(t);
+                 is_client);
-  recv_data(t, slices, nslices, GRPC_ENDPOINT_CB_OK);
 }

src/cpp/client/channel.cc

@@ -102,6 +102,7 @@ Status Channel::StartBlockingRpc(const RpcMethod &method,
   grpc_call *call = grpc_channel_create_call_old(
       c_channel_, method.name(), target_.c_str(), context->RawDeadline());
   context->set_call(call);
+
   grpc_event *ev;
   void *finished_tag = reinterpret_cast<char *>(call);
   void *metadata_read_tag = reinterpret_cast<char *>(call) + 2;

src/cpp/common/completion_queue.cc

@@ -31,7 +31,6 @@
  *
  */
 
-// TODO(yangg) maybe move to internal/common
 #include <grpc++/completion_queue.h>
 
 #include <grpc/grpc.h>

src/csharp/Grpc.sln

@@ -0,0 +1,50 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GrpcApi", "GrpcApi\GrpcApi.csproj", "{7DC1433E-3225-42C7-B7EA-546D56E27A4B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GrpcCore", "GrpcCore\GrpcCore.csproj", "{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GrpcCoreTests", "GrpcCoreTests\GrpcCoreTests.csproj", "{86EC5CB4-4EA2-40A2-8057-86542A0353BB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GrpcApiTests", "GrpcApiTests\GrpcApiTests.csproj", "{143B1C29-C442-4BE0-BF3F-A8F92288AC9F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MathClient", "MathClient\MathClient.csproj", "{61ECB8EE-0C96-4F8E-B187-8E4D227417C0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InteropClient", "InteropClient\InteropClient.csproj", "{C61154BA-DD4A-4838-8420-0162A28925E0}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|x86 = Debug|x86
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Debug|x86.Build.0 = Debug|Any CPU
+		{143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Release|x86.ActiveCfg = Release|Any CPU
+		{143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Release|x86.Build.0 = Release|Any CPU
+		{61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Debug|x86.ActiveCfg = Debug|x86
+		{61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Debug|x86.Build.0 = Debug|x86
+		{61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Release|x86.ActiveCfg = Release|x86
+		{61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Release|x86.Build.0 = Release|x86
+		{7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Debug|x86.Build.0 = Debug|Any CPU
+		{7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Release|x86.ActiveCfg = Release|Any CPU
+		{7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Release|x86.Build.0 = Release|Any CPU
+		{86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Debug|x86.Build.0 = Debug|Any CPU
+		{86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Release|x86.ActiveCfg = Release|Any CPU
+		{86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Release|x86.Build.0 = Release|Any CPU
+		{C61154BA-DD4A-4838-8420-0162A28925E0}.Debug|x86.ActiveCfg = Debug|x86
+		{C61154BA-DD4A-4838-8420-0162A28925E0}.Debug|x86.Build.0 = Debug|x86
+		{C61154BA-DD4A-4838-8420-0162A28925E0}.Release|x86.ActiveCfg = Release|x86
+		{C61154BA-DD4A-4838-8420-0162A28925E0}.Release|x86.Build.0 = Release|x86
+		{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Debug|x86.Build.0 = Debug|Any CPU
+		{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Release|x86.ActiveCfg = Release|Any CPU
+		{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Release|x86.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(MonoDevelopProperties) = preSolution
+		StartupItem = InteropClient\InteropClient.csproj
+	EndGlobalSection
+EndGlobal

src/csharp/GrpcApi/.gitignore

@@ -0,0 +1,2 @@
+test-results
+bin

src/csharp/GrpcApi/Empty.cs

@@ -0,0 +1,282 @@
+// Generated by ProtoGen, Version=2.4.1.521, Culture=neutral, PublicKeyToken=17b3b1f090c3ea48.  DO NOT EDIT!
+#pragma warning disable 1591, 0612, 3021
+#region Designer generated code
+
+using pb = global::Google.ProtocolBuffers;
+using pbc = global::Google.ProtocolBuffers.Collections;
+using pbd = global::Google.ProtocolBuffers.Descriptors;
+using scg = global::System.Collections.Generic;
+namespace grpc.testing {
+  
+  namespace Proto {
+    
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+    public static partial class Empty {
+    
+      #region Extension registration
+      public static void RegisterAllExtensions(pb::ExtensionRegistry registry) {
+      }
+      #endregion
+      #region Static variables
+      internal static pbd::MessageDescriptor internal__static_grpc_testing_Empty__Descriptor;
+      internal static pb::FieldAccess.FieldAccessorTable<global::grpc.testing.Empty, global::grpc.testing.Empty.Builder> internal__static_grpc_testing_Empty__FieldAccessorTable;
+      #endregion
+      #region Descriptor
+      public static pbd::FileDescriptor Descriptor {
+        get { return descriptor; }
+      }
+      private static pbd::FileDescriptor descriptor;
+      
+      static Empty() {
+        byte[] descriptorData = global::System.Convert.FromBase64String(
+            string.Concat(
+            "CgtlbXB0eS5wcm90bxIMZ3JwYy50ZXN0aW5nIgcKBUVtcHR5"));
+        pbd::FileDescriptor.InternalDescriptorAssigner assigner = delegate(pbd::FileDescriptor root) {
+          descriptor = root;
+          internal__static_grpc_testing_Empty__Descriptor = Descriptor.MessageTypes[0];
+          internal__static_grpc_testing_Empty__FieldAccessorTable = 
+              new pb::FieldAccess.FieldAccessorTable<global::grpc.testing.Empty, global::grpc.testing.Empty.Builder>(internal__static_grpc_testing_Empty__Descriptor,
+                  new string[] { });
+          return null;
+        };
+        pbd::FileDescriptor.InternalBuildGeneratedFileFrom(descriptorData,
+            new pbd::FileDescriptor[] {
+            }, assigner);
+      }
+      #endregion
+      
+    }
+  }
+  #region Messages
+  [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+  public sealed partial class Empty : pb::GeneratedMessage<Empty, Empty.Builder> {
+    private Empty() { }
+    private static readonly Empty defaultInstance = new Empty().MakeReadOnly();
+    private static readonly string[] _emptyFieldNames = new string[] {  };
+    private static readonly uint[] _emptyFieldTags = new uint[] {  };
+    public static Empty DefaultInstance {
+      get { return defaultInstance; }
+    }
+    
+    public override Empty DefaultInstanceForType {
+      get { return DefaultInstance; }
+    }
+    
+    protected override Empty ThisMessage {
+      get { return this; }
+    }
+    
+    public static pbd::MessageDescriptor Descriptor {
+      get { return global::grpc.testing.Proto.Empty.internal__static_grpc_testing_Empty__Descriptor; }
+    }
+    
+    protected override pb::FieldAccess.FieldAccessorTable<Empty, Empty.Builder> InternalFieldAccessors {
+      get { return global::grpc.testing.Proto.Empty.internal__static_grpc_testing_Empty__FieldAccessorTable; }
+    }
+    
+    public override bool IsInitialized {
+      get {
+        return true;
+      }
+    }
+    
+    public override void WriteTo(pb::ICodedOutputStream output) {
+      int size = SerializedSize;
+      string[] field_names = _emptyFieldNames;
+      UnknownFields.WriteTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public override int SerializedSize {
+      get {
+        int size = memoizedSerializedSize;
+        if (size != -1) return size;
+        
+        size = 0;
+        size += UnknownFields.SerializedSize;
+        memoizedSerializedSize = size;
+        return size;
+      }
+    }
+    
+    public static Empty ParseFrom(pb::ByteString data) {
+      return ((Builder) CreateBuilder().MergeFrom(data)).BuildParsed();
+    }
+    public static Empty ParseFrom(pb::ByteString data, pb::ExtensionRegistry extensionRegistry) {
+      return ((Builder) CreateBuilder().MergeFrom(data, extensionRegistry)).BuildParsed();
+    }
+    public static Empty ParseFrom(byte[] data) {
+      return ((Builder) CreateBuilder().MergeFrom(data)).BuildParsed();
+    }
+    public static Empty ParseFrom(byte[] data, pb::ExtensionRegistry extensionRegistry) {
+      return ((Builder) CreateBuilder().MergeFrom(data, extensionRegistry)).BuildParsed();
+    }
+    public static Empty ParseFrom(global::System.IO.Stream input) {
+      return ((Builder) CreateBuilder().MergeFrom(input)).BuildParsed();
+    }
+    public static Empty ParseFrom(global::System.IO.Stream input, pb::ExtensionRegistry extensionRegistry) {
+      return ((Builder) CreateBuilder().MergeFrom(input, extensionRegistry)).BuildParsed();
+    }
+    public static Empty ParseDelimitedFrom(global::System.IO.Stream input) {
+      return CreateBuilder().MergeDelimitedFrom(input).BuildParsed();
+    }
+    public static Empty ParseDelimitedFrom(global::System.IO.Stream input, pb::ExtensionRegistry extensionRegistry) {
+      return CreateBuilder().MergeDelimitedFrom(input, extensionRegistry).BuildParsed();
+    }
+    public static Empty ParseFrom(pb::ICodedInputStream input) {
+      return ((Builder) CreateBuilder().MergeFrom(input)).BuildParsed();
+    }
+    public static Empty ParseFrom(pb::ICodedInputStream input, pb::ExtensionRegistry extensionRegistry) {
+      return ((Builder) CreateBuilder().MergeFrom(input, extensionRegistry)).BuildParsed();
+    }
+    private Empty MakeReadOnly() {
+      return this;
+    }
+    
+    public static Builder CreateBuilder() { return new Builder(); }
+    public override Builder ToBuilder() { return CreateBuilder(this); }
+    public override Builder CreateBuilderForType() { return new Builder(); }
+    public static Builder CreateBuilder(Empty prototype) {
+      return new Builder(prototype);
+    }
+    
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+    public sealed partial class Builder : pb::GeneratedBuilder<Empty, Builder> {
+      protected override Builder ThisBuilder {
+        get { return this; }
+      }
+      public Builder() {
+        result = DefaultInstance;
+        resultIsReadOnly = true;
+      }
+      internal Builder(Empty cloneFrom) {
+        result = cloneFrom;
+        resultIsReadOnly = true;
+      }
+      
+      private bool resultIsReadOnly;
+      private Empty result;
+      
+      private Empty PrepareBuilder() {
+        if (resultIsReadOnly) {
+          Empty original = result;
+          result = new Empty();
+          resultIsReadOnly = false;
+          MergeFrom(original);
+        }
+        return result;
+      }
+      
+      public override bool IsInitialized {
+        get { return result.IsInitialized; }
+      }
+      
+      protected override Empty MessageBeingBuilt {
+        get { return PrepareBuilder(); }
+      }
+      
+      public override Builder Clear() {
+        result = DefaultInstance;
+        resultIsReadOnly = true;
+        return this;
+      }
+      
+      public override Builder Clone() {
+        if (resultIsReadOnly) {
+          return new Builder(result);
+        } else {
+          return new Builder().MergeFrom(result);
+        }
+      }
+      
+      public override pbd::MessageDescriptor DescriptorForType {
+        get { return global::grpc.testing.Empty.Descriptor; }
+      }
+      
+      public override Empty DefaultInstanceForType {
+        get { return global::grpc.testing.Empty.DefaultInstance; }
+      }
+      
+      public override Empty BuildPartial() {
+        if (resultIsReadOnly) {
+          return result;
+        }
+        resultIsReadOnly = true;
+        return result.MakeReadOnly();
+      }
+      
+      public override Builder MergeFrom(pb::IMessage other) {
+        if (other is Empty) {
+          return MergeFrom((Empty) other);
+        } else {
+          base.MergeFrom(other);
+          return this;
+        }
+      }
+      
+      public override Builder MergeFrom(Empty other) {
+        if (other == global::grpc.testing.Empty.DefaultInstance) return this;
+        PrepareBuilder();
+        this.MergeUnknownFields(other.UnknownFields);
+        return this;
+      }
+      
+      public override Builder MergeFrom(pb::ICodedInputStream input) {
+        return MergeFrom(input, pb::ExtensionRegistry.Empty);
+      }
+      
+      public override Builder MergeFrom(pb::ICodedInputStream input, pb::ExtensionRegistry extensionRegistry) {
+        PrepareBuilder();
+        pb::UnknownFieldSet.Builder unknownFields = null;
+        uint tag;
+        string field_name;
+        while (input.ReadTag(out tag, out field_name)) {
+          if(tag == 0 && field_name != null) {
+            int field_ordinal = global::System.Array.BinarySearch(_emptyFieldNames, field_name, global::System.StringComparer.Ordinal);
+            if(field_ordinal >= 0)
+              tag = _emptyFieldTags[field_ordinal];
+            else {
+              if (unknownFields == null) {
+                unknownFields = pb::UnknownFieldSet.CreateBuilder(this.UnknownFields);
+              }
+              ParseUnknownField(input, unknownFields, extensionRegistry, tag, field_name);
+              continue;
+            }
+          }
+          switch (tag) {
+            case 0: {
+              throw pb::InvalidProtocolBufferException.InvalidTag();
+            }
+            default: {
+              if (pb::WireFormat.IsEndGroupTag(tag)) {
+                if (unknownFields != null) {
+                  this.UnknownFields = unknownFields.Build();
+                }
+                return this;
+              }
+              if (unknownFields == null) {
+                unknownFields = pb::UnknownFieldSet.CreateBuilder(this.UnknownFields);
+              }
+              ParseUnknownField(input, unknownFields, extensionRegistry, tag, field_name);
+              break;
+            }
+          }
+        }
+        
+        if (unknownFields != null) {
+          this.UnknownFields = unknownFields.Build();
+        }
+        return this;
+      }
+      
+    }
+    static Empty() {
+      object.ReferenceEquals(global::grpc.testing.Proto.Empty.Descriptor, null);
+    }
+  }
+  
+  #endregion
+  
+}
+
+#endregion Designer generated code

src/csharp/GrpcApi/GrpcApi.csproj

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>10.0.0</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{7DC1433E-3225-42C7-B7EA-546D56E27A4B}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <RootNamespace>GrpcApi</RootNamespace>
+    <AssemblyName>GrpcApi</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>full</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Reactive.Linq, Version=2.2.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35">
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="System.Data.Linq" />
+    <Reference Include="System.Reactive.Interfaces, Version=2.2.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35">
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="System.Reactive.Core, Version=2.2.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35">
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="Google.ProtocolBuffers">
+      <HintPath>..\lib\Google.ProtocolBuffers.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Math.cs" />
+    <Compile Include="MathGrpc.cs" />
+    <Compile Include="MathServiceImpl.cs" />
+    <Compile Include="Empty.cs" />
+    <Compile Include="Messages.cs" />
+    <Compile Include="TestServiceGrpc.cs" />
+    <Compile Include="MathExamples.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+  <ItemGroup>
+    <ProjectReference Include="..\GrpcCore\GrpcCore.csproj">
+      <Project>{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}</Project>
+      <Name>GrpcCore</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="proto\math.proto" />
+    <None Include="proto\empty.proto" />
+    <None Include="proto\messages.proto" />
+    <None Include="proto\test.proto" />
+  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="proto\" />
+  </ItemGroup>
+</Project>

src/csharp/GrpcApi/MathExamples.cs

@@ -0,0 +1,98 @@
+using System;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+using Google.GRPC.Core.Utils;
+
+namespace math
+{
+	public static class MathExamples
+	{
+		public static void DivExample(MathGrpc.IMathServiceClient stub)
+		{
+			DivReply result = stub.Div(new DivArgs.Builder { Dividend = 10, Divisor = 3 }.Build());
+			Console.WriteLine("Div Result: " + result);
+		}
+
+		public static void DivAsyncExample(MathGrpc.IMathServiceClient stub)
+		{
+			Task<DivReply> call = stub.DivAsync(new DivArgs.Builder { Dividend = 4, Divisor = 5 }.Build());
+			DivReply result = call.Result;
+			Console.WriteLine(result);
+		}
+
+		public static void DivAsyncWithCancellationExample(MathGrpc.IMathServiceClient stub)
+		{
+			Task<DivReply> call = stub.DivAsync(new DivArgs.Builder { Dividend = 4, Divisor = 5 }.Build());
+			DivReply result = call.Result;
+			Console.WriteLine(result);
+		}
+
+		public static void FibExample(MathGrpc.IMathServiceClient stub)
+		{
+            var recorder = new RecordingObserver<Num>();
+            stub.Fib(new FibArgs.Builder { Limit = 5 }.Build(), recorder);
+
+			List<Num> numbers = recorder.ToList().Result;
+            Console.WriteLine("Fib Result: " + string.Join("|", recorder.ToList().Result));
+		}
+
+		public static void SumExample(MathGrpc.IMathServiceClient stub)
+		{
+			List<Num> numbers = new List<Num>{new Num.Builder { Num_ = 1 }.Build(), 
+				new Num.Builder { Num_ = 2 }.Build(),
+				new Num.Builder { Num_ = 3 }.Build()};
+			
+            var res = stub.Sum();
+            foreach (var num in numbers) {
+                res.Inputs.OnNext(num);
+            }
+            res.Inputs.OnCompleted();
+
+			Console.WriteLine("Sum Result: " + res.Task.Result);
+		}
+
+		public static void DivManyExample(MathGrpc.IMathServiceClient stub)
+		{
+			List<DivArgs> divArgsList = new List<DivArgs>{
+				new DivArgs.Builder { Dividend = 10, Divisor = 3 }.Build(),
+				new DivArgs.Builder { Dividend = 100, Divisor = 21 }.Build(),
+				new DivArgs.Builder { Dividend = 7, Divisor = 2 }.Build()
+			};
+
+            var recorder = new RecordingObserver<DivReply>();
+			
+            var inputs = stub.DivMany(recorder);
+            foreach (var input in divArgsList)
+            {
+                inputs.OnNext(input);
+            }
+            inputs.OnCompleted();
+
+			Console.WriteLine("DivMany Result: " + string.Join("|", recorder.ToList().Result));
+		}
+
+		public static void DependendRequestsExample(MathGrpc.IMathServiceClient stub)
+		{
+			var numberList = new List<Num>
+			{ new Num.Builder{ Num_ = 1 }.Build(), 
+				new Num.Builder{ Num_ = 2 }.Build(), new Num.Builder{ Num_ = 3 }.Build()
+			};
+
+			numberList.ToObservable();
+
+			//IObserver<Num> numbers;
+			//Task<Num> call = stub.Sum(out numbers);            
+			//foreach (var num in numberList)
+			//{
+			//	numbers.OnNext(num);
+			//}
+			//numbers.OnCompleted();
+
+			//Num sum = call.Result;
+
+			//DivReply result = stub.Div(new DivArgs.Builder { Dividend = sum.Num_, Divisor = numberList.Count }.Build());
+		}
+	}
+}
+

src/csharp/GrpcApi/MathGrpc.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+using Google.GRPC.Core;
+
+namespace math
+{
+    /// <summary>
+    /// Math service definitions (this is handwritten version of code that will normally be generated).
+    /// </summary>
+    public class MathGrpc
+    {
+        readonly static Marshaller<DivArgs> divArgsMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), DivArgs.ParseFrom);
+        readonly static Marshaller<DivReply> divReplyMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), DivReply.ParseFrom);
+        readonly static Marshaller<Num> numMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), Num.ParseFrom);
+        readonly static Marshaller<FibArgs> fibArgsMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), FibArgs.ParseFrom);
+
+        readonly static Method<DivArgs, DivReply> divMethod = new Method<DivArgs, DivReply>(
+            MethodType.Unary,
+            "/math.Math/Div",
+            divArgsMarshaller,
+            divReplyMarshaller
+        );
+        readonly static Method<FibArgs, Num> fibMethod = new Method<FibArgs, Num>(
+            MethodType.ServerStreaming,
+            "/math.Math/Fib",
+            fibArgsMarshaller,
+            numMarshaller
+        );
+        readonly static Method<Num, Num> sumMethod = new Method<Num, Num>(
+            MethodType.ClientStreaming,
+            "/math.Math/Sum",
+            numMarshaller,
+            numMarshaller
+        );
+        readonly static Method<DivArgs, DivReply> divManyMethod = new Method<DivArgs, DivReply>(
+            MethodType.DuplexStreaming,
+            "/math.Math/DivMany",
+            divArgsMarshaller,
+            divReplyMarshaller
+        );
+
+        public interface IMathServiceClient
+        {
+            DivReply Div(DivArgs request, CancellationToken token = default(CancellationToken));
+
+            Task<DivReply> DivAsync(DivArgs request, CancellationToken token = default(CancellationToken));
+
+            Task Fib(FibArgs request, IObserver<Num> responseObserver, CancellationToken token = default(CancellationToken));
+
+            ClientStreamingAsyncResult<Num, Num> Sum(CancellationToken token = default(CancellationToken));
+
+            IObserver<DivArgs> DivMany(IObserver<DivReply> responseObserver, CancellationToken token = default(CancellationToken));
+        }
+
+        public class MathServiceClientStub : IMathServiceClient
+        {
+            readonly Channel channel;
+
+            public MathServiceClientStub(Channel channel)
+            {
+                this.channel = channel;
+            }
+
+            public DivReply Div(DivArgs request, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<DivArgs, DivReply>(divMethod, channel);
+                return Calls.BlockingUnaryCall(call, request, token);
+            }
+
+            public Task<DivReply> DivAsync(DivArgs request, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<DivArgs, DivReply>(divMethod, channel);
+                return Calls.AsyncUnaryCall(call, request, token);
+            }
+
+            public Task Fib(FibArgs request, IObserver<Num> responseObserver, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<FibArgs, Num>(fibMethod, channel);
+                return Calls.AsyncServerStreamingCall(call, request, responseObserver, token);
+            }
+
+            public ClientStreamingAsyncResult<Num, Num> Sum(CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<Num, Num>(sumMethod, channel);
+                return Calls.AsyncClientStreamingCall(call, token);
+            }
+
+            public IObserver<DivArgs> DivMany(IObserver<DivReply> responseObserver, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<DivArgs, DivReply>(divManyMethod, channel);
+                return Calls.DuplexStreamingCall(call, responseObserver, token);
+            }
+        }
+
+        // server-side interface
+        public interface IMathService
+        {
+            void Div(DivArgs request, IObserver<DivReply> responseObserver);
+
+            void Fib(FibArgs request, IObserver<Num> responseObserver);
+
+            IObserver<Num> Sum(IObserver<Num> responseObserver);
+
+            IObserver<DivArgs> DivMany(IObserver<DivReply> responseObserver);
+        }
+
+        public static ServerServiceDefinition BindService(IMathService serviceImpl)
+        {
+            return ServerServiceDefinition.CreateBuilder("/math.Math/")
+                .AddMethod(divMethod, serviceImpl.Div)
+                .AddMethod(fibMethod, serviceImpl.Fib)
+                .AddMethod(sumMethod, serviceImpl.Sum)
+                .AddMethod(divManyMethod, serviceImpl.DivMany).Build();
+        }
+
+        public static IMathServiceClient NewStub(Channel channel)
+        {
+            return new MathServiceClientStub(channel);
+        }
+    }
+}

src/csharp/GrpcApi/MathServiceImpl.cs

@@ -0,0 +1,119 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+using Google.GRPC.Core.Utils;
+
+namespace math
+{
+    /// <summary>
+    /// Implementation of MathService server
+    /// </summary>
+    public class MathServiceImpl : MathGrpc.IMathService
+    {
+        public void Div(DivArgs request, IObserver<DivReply> responseObserver)
+        {
+            var response = DivInternal(request);
+            responseObserver.OnNext(response);
+            responseObserver.OnCompleted();
+        }
+
+        public void Fib(FibArgs request, IObserver<Num> responseObserver)
+        {
+            if (request.Limit <= 0)
+            {
+                // TODO: support cancellation....
+                throw new NotImplementedException("Not implemented yet");
+            }
+                                  
+            if (request.Limit > 0)
+            {
+                foreach (var num in FibInternal(request.Limit))
+                {
+                    responseObserver.OnNext(num);
+                }
+                responseObserver.OnCompleted();
+            }
+        }
+
+        public IObserver<Num> Sum(IObserver<Num> responseObserver)
+        {
+            var recorder = new RecordingObserver<Num>();
+            Task.Factory.StartNew(() => {
+
+                List<Num> inputs = recorder.ToList().Result;
+
+                long sum = 0;
+                foreach (Num num in inputs)
+                {
+                    sum += num.Num_;
+                }
+
+                responseObserver.OnNext(Num.CreateBuilder().SetNum_(sum).Build());
+                responseObserver.OnCompleted();
+            });
+            return recorder;
+        }
+
+        public IObserver<DivArgs> DivMany(IObserver<DivReply> responseObserver)
+        {
+            return new DivObserver(responseObserver);
+        }
+
+        static DivReply DivInternal(DivArgs args)
+        {
+            long quotient = args.Dividend / args.Divisor;
+            long remainder = args.Dividend % args.Divisor;
+            return new DivReply.Builder { Quotient = quotient, Remainder = remainder }.Build();
+        }
+
+        static IEnumerable<Num> FibInternal(long n)
+        {
+            long a = 1;
+            yield return new Num.Builder { Num_=a }.Build();
+
+            long b = 1;
+            for (long i = 0; i < n - 1; i++)
+            {
+                long temp = a;
+                a = b;
+                b = temp + b;
+                yield return new Num.Builder { Num_=a }.Build();
+            }
+        }
+
+        private class DivObserver : IObserver<DivArgs> {
+
+            readonly IObserver<DivReply> responseObserver;
+
+            public DivObserver(IObserver<DivReply> responseObserver)
+            {
+                this.responseObserver = responseObserver;
+            }
+            
+            public void OnCompleted()
+            {
+                Task.Factory.StartNew(() =>
+                    responseObserver.OnCompleted());
+            }
+
+            public void OnError(Exception error)
+            {
+                throw new NotImplementedException();
+            }
+
+            public void OnNext(DivArgs value)
+            {
+                // TODO: currently we need this indirection because
+                // responseObserver waits for write to finish, this
+                // callback is called from grpc threadpool which
+                // currently only has one thread.
+                // Same story for OnCompleted().
+                Task.Factory.StartNew(() => 
+                responseObserver.OnNext(DivInternal(value)));
+            }
+        }
+    }
+}
+

src/csharp/GrpcApi/Properties/AssemblyInfo.cs

@@ -0,0 +1,22 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+// Information about this assembly is defined by the following attributes. 
+// Change them to the values specific to your project.
+[assembly: AssemblyTitle ("GrpcApi")]
+[assembly: AssemblyDescription ("")]
+[assembly: AssemblyConfiguration ("")]
+[assembly: AssemblyCompany ("")]
+[assembly: AssemblyProduct ("")]
+[assembly: AssemblyCopyright ("jtattermusch")]
+[assembly: AssemblyTrademark ("")]
+[assembly: AssemblyCulture ("")]
+// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
+// The form "{Major}.{Minor}.*" will automatically update the build and revision,
+// and "{Major}.{Minor}.{Build}.*" will update just the revision.
+[assembly: AssemblyVersion ("1.0.*")]
+// The following attributes are used to specify the signing key for the assembly, 
+// if desired. See the Mono documentation for more information about signing.
+//[assembly: AssemblyDelaySign(false)]
+//[assembly: AssemblyKeyFile("")]
+

src/csharp/GrpcApi/TestServiceGrpc.cs

@@ -0,0 +1,170 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+using Google.GRPC.Core;
+
+namespace grpc.testing
+{
+    /// <summary>
+    /// TestService (this is handwritten version of code that will normally be generated).
+    /// </summary>
+    public class TestServiceGrpc
+    {
+        readonly static Marshaller<Empty> emptyMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), Empty.ParseFrom);
+        readonly static Marshaller<SimpleRequest> simpleRequestMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), SimpleRequest.ParseFrom);
+        readonly static Marshaller<SimpleResponse> simpleResponseMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), SimpleResponse.ParseFrom);
+        readonly static Marshaller<StreamingOutputCallRequest> streamingOutputCallRequestMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), StreamingOutputCallRequest.ParseFrom);
+        readonly static Marshaller<StreamingOutputCallResponse> streamingOutputCallResponseMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), StreamingOutputCallResponse.ParseFrom);
+        readonly static Marshaller<StreamingInputCallRequest> streamingInputCallRequestMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), StreamingInputCallRequest.ParseFrom);
+        readonly static Marshaller<StreamingInputCallResponse> streamingInputCallResponseMarshaller = Marshallers.Create((arg) => arg.ToByteArray(), StreamingInputCallResponse.ParseFrom);
+
+        readonly static Method<Empty, Empty> emptyCallMethod = new Method<Empty, Empty>(
+            MethodType.Unary,
+            "/grpc.testing.TestService/EmptyCall",
+            emptyMarshaller,
+            emptyMarshaller
+        );
+        readonly static Method<SimpleRequest, SimpleResponse> unaryCallMethod = new Method<SimpleRequest, SimpleResponse>(
+            MethodType.Unary,
+            "/grpc.testing.TestService/UnaryCall",
+            simpleRequestMarshaller,
+            simpleResponseMarshaller
+        );
+        readonly static Method<StreamingOutputCallRequest, StreamingOutputCallResponse> streamingOutputCallMethod = new Method<StreamingOutputCallRequest, StreamingOutputCallResponse>(
+            MethodType.ServerStreaming,
+            "/grpc.testing.TestService/StreamingOutputCall",
+            streamingOutputCallRequestMarshaller,
+            streamingOutputCallResponseMarshaller
+            );
+        readonly static Method<StreamingInputCallRequest, StreamingInputCallResponse> streamingInputCallMethod = new Method<StreamingInputCallRequest, StreamingInputCallResponse>(
+            MethodType.ClientStreaming,
+            "/grpc.testing.TestService/StreamingInputCall",
+            streamingInputCallRequestMarshaller,
+            streamingInputCallResponseMarshaller
+            );
+        readonly static Method<StreamingOutputCallRequest, StreamingOutputCallResponse> fullDuplexCallMethod = new Method<StreamingOutputCallRequest, StreamingOutputCallResponse>(
+            MethodType.DuplexStreaming,
+            "/grpc.testing.TestService/FullDuplexCall",
+            streamingOutputCallRequestMarshaller,
+            streamingOutputCallResponseMarshaller
+            );
+        readonly static Method<StreamingOutputCallRequest, StreamingOutputCallResponse> halfDuplexCallMethod = new Method<StreamingOutputCallRequest, StreamingOutputCallResponse>(
+            MethodType.DuplexStreaming,
+            "/grpc.testing.TestService/HalfDuplexCall",
+            streamingOutputCallRequestMarshaller,
+            streamingOutputCallResponseMarshaller
+            );
+
+        public interface ITestServiceClient
+        {
+            Empty EmptyCall(Empty request, CancellationToken token = default(CancellationToken));
+
+            Task<Empty> EmptyCallAsync(Empty request, CancellationToken token = default(CancellationToken));
+
+            SimpleResponse UnaryCall(SimpleRequest request, CancellationToken token = default(CancellationToken));
+
+            Task<SimpleResponse> UnaryCallAsync(SimpleRequest request, CancellationToken token = default(CancellationToken));
+
+            Task StreamingOutputCall(StreamingOutputCallRequest request, IObserver<StreamingOutputCallResponse> responseObserver, CancellationToken token = default(CancellationToken));
+
+            ClientStreamingAsyncResult<StreamingInputCallRequest, StreamingInputCallResponse> StreamingInputCall(CancellationToken token = default(CancellationToken));
+
+            IObserver<StreamingOutputCallRequest> FullDuplexCall(IObserver<StreamingOutputCallResponse> responseObserver, CancellationToken token = default(CancellationToken));
+
+            IObserver<StreamingOutputCallRequest> HalfDuplexCall(IObserver<StreamingOutputCallResponse> responseObserver, CancellationToken token = default(CancellationToken));
+        }
+
+        public class TestServiceClientStub : ITestServiceClient
+        {
+            readonly Channel channel;
+
+            public TestServiceClientStub(Channel channel)
+            {
+                this.channel = channel;
+            }
+
+            public Empty EmptyCall(Empty request, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<Empty, Empty>(emptyCallMethod, channel);
+                return Calls.BlockingUnaryCall(call, request, token);
+            }
+
+            public Task<Empty> EmptyCallAsync(Empty request, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<Empty, Empty>(emptyCallMethod, channel);
+                return Calls.AsyncUnaryCall(call, request, token);
+            }
+
+            public SimpleResponse UnaryCall(SimpleRequest request, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<SimpleRequest, SimpleResponse>(unaryCallMethod, channel);
+                return Calls.BlockingUnaryCall(call, request, token);
+            }
+
+            public Task<SimpleResponse> UnaryCallAsync(SimpleRequest request, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<SimpleRequest, SimpleResponse>(unaryCallMethod, channel);
+                return Calls.AsyncUnaryCall(call, request, token);
+            }
+
+            public Task StreamingOutputCall(StreamingOutputCallRequest request, IObserver<StreamingOutputCallResponse> responseObserver, CancellationToken token = default(CancellationToken)) {
+                var call = new Google.GRPC.Core.Call<StreamingOutputCallRequest, StreamingOutputCallResponse>(streamingOutputCallMethod, channel);
+                return Calls.AsyncServerStreamingCall(call, request, responseObserver, token);
+            }
+
+            public ClientStreamingAsyncResult<StreamingInputCallRequest, StreamingInputCallResponse> StreamingInputCall(CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<StreamingInputCallRequest, StreamingInputCallResponse>(streamingInputCallMethod, channel);
+                return Calls.AsyncClientStreamingCall(call, token);
+            }
+
+            public IObserver<StreamingOutputCallRequest> FullDuplexCall(IObserver<StreamingOutputCallResponse> responseObserver, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<StreamingOutputCallRequest, StreamingOutputCallResponse>(fullDuplexCallMethod, channel);
+                return Calls.DuplexStreamingCall(call, responseObserver, token);
+            }
+
+
+            public IObserver<StreamingOutputCallRequest> HalfDuplexCall(IObserver<StreamingOutputCallResponse> responseObserver, CancellationToken token = default(CancellationToken))
+            {
+                var call = new Google.GRPC.Core.Call<StreamingOutputCallRequest, StreamingOutputCallResponse>(halfDuplexCallMethod, channel);
+                return Calls.DuplexStreamingCall(call, responseObserver, token);
+            }
+        }
+
+        // server-side interface
+        public interface ITestService
+        {
+            void EmptyCall(Empty request, IObserver<Empty> responseObserver);
+
+            void UnaryCall(SimpleRequest request, IObserver<SimpleResponse> responseObserver);
+
+            void StreamingOutputCall(StreamingOutputCallRequest request, IObserver<StreamingOutputCallResponse> responseObserver);
+
+            IObserver<StreamingInputCallRequest> StreamingInputCall(IObserver<StreamingInputCallResponse> responseObserver);
+
+            IObserver<StreamingOutputCallRequest> FullDuplexCall(IObserver<StreamingOutputCallResponse> responseObserver);
+
+            IObserver<StreamingOutputCallRequest> HalfDuplexCall(IObserver<StreamingOutputCallResponse> responseObserver);
+        }
+
+        public static ServerServiceDefinition BindService(ITestService serviceImpl)
+        {
+            return ServerServiceDefinition.CreateBuilder("/grpc.testing.TestService/")
+                .AddMethod(emptyCallMethod, serviceImpl.EmptyCall)
+                .AddMethod(unaryCallMethod, serviceImpl.UnaryCall)
+                .AddMethod(streamingOutputCallMethod, serviceImpl.StreamingOutputCall)
+                .AddMethod(streamingInputCallMethod, serviceImpl.StreamingInputCall)
+                .AddMethod(fullDuplexCallMethod, serviceImpl.FullDuplexCall)
+                .AddMethod(halfDuplexCallMethod, serviceImpl.HalfDuplexCall)
+                .Build();
+        }
+
+        public static ITestServiceClient NewStub(Channel channel)
+        {
+            return new TestServiceClientStub(channel);
+        }
+    }
+}

src/csharp/GrpcApi/proto/empty.proto

@@ -0,0 +1,13 @@
+syntax = "proto2";
+
+package grpc.testing;
+
+// An empty message that you can re-use to avoid defining duplicated empty
+// messages in your project. A typical example is to use it as argument or the
+// return value of a service API. For instance:
+//
+//   service Foo {
+//     rpc Bar (grpc.testing.Empty) returns (grpc.testing.Empty) { };
+//   };
+//
+message Empty {}

src/csharp/GrpcApi/proto/math.proto

@@ -0,0 +1,50 @@
+syntax = "proto2";
+
+package math;
+
+message DivArgs {
+  optional int64 dividend = 1;
+  optional int64 divisor = 2;
+}
+
+message DivReply {
+  optional int64 quotient = 1;
+  optional int64 remainder = 2;
+}
+
+message FibArgs {
+  optional int64 limit = 1;
+}
+
+message Num {
+  optional int64 num = 1;
+}
+
+message FibReply {
+  optional int64 count = 1;
+}
+
+service Math {
+  // Div divides args.dividend by args.divisor and returns the quotient and
+  // remainder.
+  rpc Div (DivArgs) returns (DivReply) {
+  }
+
+  // DivMany accepts an arbitrary number of division args from the client stream
+  // and sends back the results in the reply stream.  The stream continues until
+  // the client closes its end; the server does the same after sending all the
+  // replies.  The stream ends immediately if either end aborts.
+  rpc DivMany (stream DivArgs) returns (stream DivReply) {
+  }
+
+  // Fib generates numbers in the Fibonacci sequence.  If args.limit > 0, Fib
+  // generates up to limit numbers; otherwise it continues until the call is
+  // canceled.  Unlike Fib above, Fib has no final FibReply.
+  rpc Fib (FibArgs) returns (stream Num) {
+  }
+
+  // Sum sums a stream of numbers, returning the final result once the stream
+  // is closed.
+  rpc Sum (stream Num) returns (Num) {
+  }
+}

src/csharp/GrpcApi/proto/messages.proto

@@ -0,0 +1,102 @@
+// Message definitions to be used by integration test service definitions.
+
+syntax = "proto2";
+
+package grpc.testing;
+
+// The type of payload that should be returned.
+enum PayloadType {
+  // Compressable text format.
+  COMPRESSABLE = 0;
+
+  // Uncompressable binary format.
+  UNCOMPRESSABLE = 1;
+
+  // Randomly chosen from all other formats defined in this enum.
+  RANDOM = 2;
+}
+
+// A block of data, to simply increase gRPC message size.
+message Payload {
+  // The type of data in body.
+  optional PayloadType type = 1;
+  // Primary contents of payload.
+  optional bytes body = 2;
+}
+
+// Unary request.
+message SimpleRequest {
+  // Desired payload type in the response from the server.
+  // If response_type is RANDOM, server randomly chooses one from other formats.
+  optional PayloadType response_type = 1;
+
+  // Desired payload size in the response from the server.
+  // If response_type is COMPRESSABLE, this denotes the size before compression.
+  optional int32 response_size = 2;
+
+  // Optional input payload sent along with the request.
+  optional Payload payload = 3;
+
+  // Whether SimpleResponse should include username.
+  optional bool fill_username = 4;
+
+  // Whether SimpleResponse should include OAuth scope.
+  optional bool fill_oauth_scope = 5;
+}
+
+// Unary response, as configured by the request.
+message SimpleResponse {
+  // Payload to increase message size.
+  optional Payload payload = 1;
+  // The user the request came from, for verifying authentication was
+  // successful when the client expected it.
+  optional string username = 2;
+  // OAuth scope.
+  optional string oauth_scope = 3;
+}
+
+// Client-streaming request.
+message StreamingInputCallRequest {
+  // Optional input payload sent along with the request.
+  optional Payload payload = 1;
+
+  // Not expecting any payload from the response.
+}
+
+// Client-streaming response.
+message StreamingInputCallResponse {
+  // Aggregated size of payloads received from the client.
+  optional int32 aggregated_payload_size = 1;
+}
+
+// Configuration for a particular response.
+message ResponseParameters {
+  // Desired payload sizes in responses from the server.
+  // If response_type is COMPRESSABLE, this denotes the size before compression.
+  optional int32 size = 1;
+
+  // Desired interval between consecutive responses in the response stream in
+  // microseconds.
+  optional int32 interval_us = 2;
+}
+
+// Server-streaming request.
+message StreamingOutputCallRequest {
+  // Desired payload type in the response from the server.
+  // If response_type is RANDOM, the payload from each response in the stream
+  // might be of different types. This is to simulate a mixed type of payload
+  // stream.
+  optional PayloadType response_type = 1;
+
+  // Configuration for each expected response message.
+  repeated ResponseParameters response_parameters = 2;
+
+  // Optional input payload sent along with the request.
+  optional Payload payload = 3;
+}
+
+// Server-streaming response, as configured by the request and parameters.
+message StreamingOutputCallResponse {
+  // Payload to increase response size.
+  optional Payload payload = 1;
+}

src/csharp/GrpcApi/proto/test.proto

@@ -0,0 +1,42 @@
+// An integration test service that covers all the method signature permutations
+// of unary/streaming requests/responses.
+syntax = "proto2";
+
+import "empty.proto";
+import "messages.proto";
+
+package grpc.testing;
+
+// A simple service to test the various types of RPCs and experiment with
+// performance with various types of payload.
+service TestService {
+  // One empty request followed by one empty response.
+  rpc EmptyCall(grpc.testing.Empty) returns (grpc.testing.Empty);
+
+  // One request followed by one response.
+  // The server returns the client payload as-is.
+  rpc UnaryCall(SimpleRequest) returns (SimpleResponse);
+
+  // One request followed by a sequence of responses (streamed download).
+  // The server returns the payload with client desired type and sizes.
+  rpc StreamingOutputCall(StreamingOutputCallRequest)
+      returns (stream StreamingOutputCallResponse);
+
+  // A sequence of requests followed by one response (streamed upload).
+  // The server returns the aggregated size of client payload as the result.
+  rpc StreamingInputCall(stream StreamingInputCallRequest)
+      returns (StreamingInputCallResponse);
+
+  // A sequence of requests with each request served by the server immediately.
+  // As one request could lead to multiple responses, this interface
+  // demonstrates the idea of full duplexing.
+  rpc FullDuplexCall(stream StreamingOutputCallRequest)
+      returns (stream StreamingOutputCallResponse);
+
+  // A sequence of requests followed by a sequence of responses.
+  // The server buffers all the client requests and then serves them in order. A
+  // stream of responses are returned to the client when the server starts with
+  // first request.
+  rpc HalfDuplexCall(stream StreamingOutputCallRequest)
+      returns (stream StreamingOutputCallResponse);
+}

src/csharp/GrpcApiTests/.gitignore

@@ -0,0 +1,2 @@
+test-results
+bin

src/csharp/GrpcApiTests/GrpcApiTests.csproj

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>10.0.0</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{143B1C29-C442-4BE0-BF3F-A8F92288AC9F}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <RootNamespace>GrpcApiTests</RootNamespace>
+    <AssemblyName>GrpcApiTests</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>full</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="nunit.framework, Version=2.6.0.0, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77">
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="Google.ProtocolBuffers">
+      <HintPath>..\lib\Google.ProtocolBuffers.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="MathClientServerTests.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+  <ItemGroup>
+    <ProjectReference Include="..\GrpcApi\GrpcApi.csproj">
+      <Project>{7DC1433E-3225-42C7-B7EA-546D56E27A4B}</Project>
+      <Name>GrpcApi</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\GrpcCore\GrpcCore.csproj">
+      <Project>{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}</Project>
+      <Name>GrpcCore</Name>
+    </ProjectReference>
+  </ItemGroup>
+</Project>

src/csharp/GrpcApiTests/MathClientServerTests.cs

@@ -0,0 +1,115 @@
+using System;
+using NUnit.Framework;
+using Google.GRPC.Core;
+using System.Threading;
+using System.Threading.Tasks;
+using Google.GRPC.Core.Utils;
+using System.Collections.Generic;
+
+namespace math.Tests
+{
+    /// <summary>
+    /// Math client talks to local math server.
+    /// </summary>
+    public class MathClientServerTest
+    {
+        string serverAddr = "localhost:" + PortPicker.PickUnusedPort();
+        Server server;
+        Channel channel;
+        MathGrpc.IMathServiceClient client;
+
+        [TestFixtureSetUp]
+        public void Init()
+        {
+            server = new Server();
+            server.AddServiceDefinition(MathGrpc.BindService(new MathServiceImpl()));
+            server.AddPort(serverAddr);
+            server.Start();
+            channel = new Channel(serverAddr);
+            client = MathGrpc.NewStub(channel);
+        }
+
+        [Test]
+        public void Div1()
+        {
+            DivReply response = client.Div(new DivArgs.Builder { Dividend = 10, Divisor = 3 }.Build());
+            Assert.AreEqual(3, response.Quotient);
+            Assert.AreEqual(1, response.Remainder);
+        }
+
+        [Test]
+        public void Div2()
+        {
+            DivReply response = client.Div(new DivArgs.Builder { Dividend = 0, Divisor = 1 }.Build());
+            Assert.AreEqual(0, response.Quotient);
+            Assert.AreEqual(0, response.Remainder);
+        }
+
+        // TODO: test division by zero
+
+        [Test]
+        public void DivAsync()
+        {
+            DivReply response = client.DivAsync(new DivArgs.Builder { Dividend = 10, Divisor = 3 }.Build()).Result;
+            Assert.AreEqual(3, response.Quotient);
+            Assert.AreEqual(1, response.Remainder);
+        }
+
+        [Test]
+        public void Fib()
+        {
+            var recorder = new RecordingObserver<Num>();
+            client.Fib(new FibArgs.Builder { Limit = 6 }.Build(), recorder);
+
+            CollectionAssert.AreEqual(new List<long>{1, 1, 2, 3, 5, 8}, 
+                recorder.ToList().Result.ConvertAll((n) => n.Num_));
+        }
+
+        // TODO: test Fib with limit=0 and cancellation
+        [Test]
+        public void Sum()
+        {
+            var res = client.Sum();
+            foreach (var num in new long[] { 10, 20, 30 }) {
+                res.Inputs.OnNext(Num.CreateBuilder().SetNum_(num).Build());
+            }
+            res.Inputs.OnCompleted();
+
+            Assert.AreEqual(60, res.Task.Result.Num_);
+        }
+
+        [Test]
+        public void DivMany()
+        {
+            List<DivArgs> divArgsList = new List<DivArgs>{
+                new DivArgs.Builder { Dividend = 10, Divisor = 3 }.Build(),
+                new DivArgs.Builder { Dividend = 100, Divisor = 21 }.Build(),
+                new DivArgs.Builder { Dividend = 7, Divisor = 2 }.Build()
+            };
+
+            var recorder = new RecordingObserver<DivReply>();
+            var requestObserver = client.DivMany(recorder);
+
+            foreach (var arg in divArgsList)
+            {
+                requestObserver.OnNext(arg);
+            }
+            requestObserver.OnCompleted();
+
+            var result = recorder.ToList().Result;
+
+            CollectionAssert.AreEqual(new long[] {3, 4, 3}, result.ConvertAll((divReply) => divReply.Quotient));
+            CollectionAssert.AreEqual(new long[] {1, 16, 1}, result.ConvertAll((divReply) => divReply.Remainder));
+        }
+
+        [TestFixtureTearDown]
+        public void Cleanup()
+        {
+            channel.Dispose();
+
+            server.ShutdownAsync().Wait();
+            GrpcEnvironment.Shutdown();
+        }
+    }
+}
+

src/csharp/GrpcApiTests/Properties/AssemblyInfo.cs

@@ -0,0 +1,22 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+// Information about this assembly is defined by the following attributes. 
+// Change them to the values specific to your project.
+[assembly: AssemblyTitle("GrpcApiTests")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("")]
+[assembly: AssemblyCopyright("jtattermusch")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
+// The form "{Major}.{Minor}.*" will automatically update the build and revision,
+// and "{Major}.{Minor}.{Build}.*" will update just the revision.
+[assembly: AssemblyVersion("1.0.*")]
+// The following attributes are used to specify the signing key for the assembly, 
+// if desired. See the Mono documentation for more information about signing.
+//[assembly: AssemblyDelaySign(false)]
+//[assembly: AssemblyKeyFile("")]
+

src/csharp/GrpcCore/.gitignore

@@ -0,0 +1 @@
+bin

src/csharp/GrpcCore/Call.cs

@@ -0,0 +1,65 @@
+using System;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core
+{
+    public class Call<TRequest, TResponse>
+    {
+        readonly string methodName;
+        readonly Func<TRequest, byte[]> requestSerializer;
+        readonly Func<byte[], TResponse> responseDeserializer;
+        readonly Channel channel;
+
+        public Call(string methodName, 
+                    Func<TRequest, byte[]> requestSerializer,
+                    Func<byte[], TResponse> responseDeserializer,
+                    TimeSpan timeout,
+                    Channel channel) {
+            this.methodName = methodName;
+            this.requestSerializer = requestSerializer;
+            this.responseDeserializer = responseDeserializer;
+            this.channel = channel;
+        }
+
+        public Call(Method<TRequest, TResponse> method, Channel channel)
+        {
+            this.methodName = method.Name;
+            this.requestSerializer = method.RequestMarshaller.Serializer;
+            this.responseDeserializer = method.ResponseMarshaller.Deserializer;
+            this.channel = channel;
+        }
+
+        public Channel Channel
+        {
+            get
+            {
+                return this.channel;
+            }
+        }
+
+        public string MethodName
+        {
+            get
+            {
+                return this.methodName;
+            }
+        }
+
+        public Func<TRequest, byte[]> RequestSerializer
+        {
+            get
+            {
+                return this.requestSerializer;
+            }
+        }
+
+        public Func<byte[], TResponse> ResponseDeserializer
+        {
+            get
+            {
+                return this.responseDeserializer;
+            }
+        }
+    }
+}
+

src/csharp/GrpcCore/Calls.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core
+{
+    // NOTE: this class is work-in-progress
+
+    /// <summary>
+    /// Helper methods for generated stubs to make RPC calls.
+    /// </summary>
+    public static class Calls
+    {
+        public static TResponse BlockingUnaryCall<TRequest, TResponse>(Call<TRequest, TResponse> call, TRequest req, CancellationToken token)
+        {
+            //TODO: implement this in real synchronous style once new GRPC C core API is available.
+            return AsyncUnaryCall(call, req, token).Result;
+        }
+
+        public static async Task<TResponse> AsyncUnaryCall<TRequest, TResponse>(Call<TRequest, TResponse> call, TRequest req, CancellationToken token)
+        {
+            var asyncCall = new AsyncCall<TRequest, TResponse>(call.RequestSerializer, call.ResponseDeserializer);
+            asyncCall.Initialize(call.Channel, call.MethodName);
+            asyncCall.Start(false, GetCompletionQueue());
+
+            await asyncCall.WriteAsync(req);
+            await asyncCall.WritesCompletedAsync();
+
+            TResponse response = await asyncCall.ReadAsync();
+
+            Status status = await asyncCall.Finished;
+
+            if (status.StatusCode != StatusCode.GRPC_STATUS_OK)
+            {
+                throw new RpcException(status);
+            }
+            return response;
+        }
+
+        public static async Task AsyncServerStreamingCall<TRequest, TResponse>(Call<TRequest, TResponse> call, TRequest req, IObserver<TResponse> outputs, CancellationToken token)
+        {
+            var asyncCall = new AsyncCall<TRequest, TResponse>(call.RequestSerializer, call.ResponseDeserializer);
+            asyncCall.Initialize(call.Channel, call.MethodName);
+            asyncCall.Start(false, GetCompletionQueue());
+
+            asyncCall.StartReadingToStream(outputs);
+
+            await asyncCall.WriteAsync(req);
+            await asyncCall.WritesCompletedAsync();
+        }
+
+        public static ClientStreamingAsyncResult<TRequest, TResponse> AsyncClientStreamingCall<TRequest, TResponse>(Call<TRequest, TResponse> call, CancellationToken token)
+        {
+            var asyncCall = new AsyncCall<TRequest, TResponse>(call.RequestSerializer, call.ResponseDeserializer);
+            asyncCall.Initialize(call.Channel, call.MethodName);
+            asyncCall.Start(false, GetCompletionQueue());
+
+            var task = asyncCall.ReadAsync();
+            var inputs = new StreamingInputObserver<TRequest, TResponse>(asyncCall);
+            return new ClientStreamingAsyncResult<TRequest, TResponse>(task, inputs);
+        }
+
+        public static TResponse BlockingClientStreamingCall<TRequest, TResponse>(Call<TRequest, TResponse> call, IObservable<TRequest> inputs, CancellationToken token)
+        {
+            throw new NotImplementedException();
+        }
+
+        public static IObserver<TRequest> DuplexStreamingCall<TRequest, TResponse>(Call<TRequest, TResponse> call, IObserver<TResponse> outputs, CancellationToken token)
+        {
+            var asyncCall = new AsyncCall<TRequest, TResponse>(call.RequestSerializer, call.ResponseDeserializer);
+            asyncCall.Initialize(call.Channel, call.MethodName);
+            asyncCall.Start(false, GetCompletionQueue());
+
+            asyncCall.StartReadingToStream(outputs);
+            var inputs = new StreamingInputObserver<TRequest, TResponse>(asyncCall);
+            return inputs;
+        }
+
+        private static CompletionQueueSafeHandle GetCompletionQueue() {
+            return GrpcEnvironment.ThreadPool.CompletionQueue;
+        }
+    }
+}
+

src/csharp/GrpcCore/Channel.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core
+{
+	public class Channel : IDisposable
+	{
+        /// <summary>
+        /// Make sure GPRC environment is initialized before any channels get used.
+        /// </summary>
+        static Channel() {
+            GrpcEnvironment.EnsureInitialized();
+        }
+       
+        readonly ChannelSafeHandle handle;
+        readonly String target;
+
+        // TODO: add way how to create grpc_secure_channel....
+		// TODO: add support for channel args...
+		public Channel(string target)
+		{
+            this.handle = ChannelSafeHandle.Create(target, IntPtr.Zero);
+			this.target = target;
+		}
+
+        internal ChannelSafeHandle Handle
+        {
+            get
+            {
+                return this.handle;
+            }
+        }
+
+        public string Target
+        {
+            get
+            {
+                return this.target;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (handle != null && !handle.IsInvalid)
+            {
+                handle.Dispose();
+            }
+        }
+	}
+}

src/csharp/GrpcCore/ClientStreamingAsyncResult.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Threading.Tasks;
+
+namespace Google.GRPC.Core
+{
+    /// <summary>
+    /// Return type for client streaming async method.
+    /// </summary>
+    public struct ClientStreamingAsyncResult<TRequest, TResponse>
+    {
+        readonly Task<TResponse> task;
+        readonly IObserver<TRequest> inputs;
+
+        public ClientStreamingAsyncResult(Task<TResponse> task, IObserver<TRequest> inputs)
+        {
+            this.task = task;
+            this.inputs = inputs;
+        }
+
+        public Task<TResponse> Task
+        {
+            get
+            {
+                return this.task;
+            }
+        }
+
+        public IObserver<TRequest> Inputs
+        {
+            get
+            {
+                return this.inputs;
+            }
+        }
+    }
+}
+

src/csharp/GrpcCore/GrpcCore.csproj

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>10.0.0</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <RootNamespace>GrpcCore</RootNamespace>
+    <AssemblyName>GrpcCore</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>full</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="RpcException.cs" />
+    <Compile Include="Calls.cs" />
+    <Compile Include="Call.cs" />
+    <Compile Include="ClientStreamingAsyncResult.cs" />
+    <Compile Include="GrpcEnvironment.cs" />
+    <Compile Include="Status.cs" />
+    <Compile Include="StatusCode.cs" />
+    <Compile Include="Server.cs" />
+    <Compile Include="Channel.cs" />
+    <Compile Include="Internal\CallSafeHandle.cs" />
+    <Compile Include="Internal\ChannelSafeHandle.cs" />
+    <Compile Include="Internal\CompletionQueueSafeHandle.cs" />
+    <Compile Include="Internal\Enums.cs" />
+    <Compile Include="Internal\Event.cs" />
+    <Compile Include="Internal\SafeHandleZeroIsInvalid.cs" />
+    <Compile Include="Internal\Timespec.cs" />
+    <Compile Include="Internal\GrpcThreadPool.cs" />
+    <Compile Include="Internal\AsyncCall.cs" />
+    <Compile Include="Internal\ServerSafeHandle.cs" />
+    <Compile Include="Internal\StreamingInputObserver.cs" />
+    <Compile Include="Method.cs" />
+    <Compile Include="ServerCalls.cs" />
+    <Compile Include="ServerCallHandler.cs" />
+    <Compile Include="Internal\ServerWritingObserver.cs" />
+    <Compile Include="Marshaller.cs" />
+    <Compile Include="ServerServiceDefinition.cs" />
+    <Compile Include="Utils\RecordingObserver.cs" />
+    <Compile Include="Utils\PortPicker.cs" />
+    <Compile Include="Utils\RecordingQueue.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+  <ItemGroup>
+    <Folder Include="Internal\" />
+    <Folder Include="Utils\" />
+  </ItemGroup>
+</Project>

src/csharp/GrpcCore/GrpcEnvironment.cs

@@ -0,0 +1,91 @@
+using System;
+using Google.GRPC.Core.Internal;
+using System.Runtime.InteropServices;
+
+namespace Google.GRPC.Core
+{
+    /// <summary>
+    /// Encapsulates initialization and shutdown of GRPC C core library.
+    /// You should not need to initialize it manually, as static constructors
+    /// should load the library when needed.
+    /// </summary>
+    public static class GrpcEnvironment
+    {
+        const int THREAD_POOL_SIZE = 1;
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_init();
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_shutdown();
+
+        static object staticLock = new object();
+        static bool initCalled = false;
+        static bool shutdownCalled = false;
+
+        static GrpcThreadPool threadPool = new GrpcThreadPool(THREAD_POOL_SIZE);
+
+        /// <summary>
+        /// Makes sure GRPC environment is initialized.
+        /// </summary>
+        public static void EnsureInitialized() {
+            lock(staticLock)
+            {
+                if (!initCalled)
+                {
+                    initCalled = true;
+                    GrpcInit();       
+                }
+            }
+        }
+
+        /// <summary>
+        /// Shuts down the GRPC environment if it was initialized before.
+        /// Repeated invocations have no effect.
+        /// </summary>
+        public static void Shutdown()
+        {
+            lock(staticLock)
+            {
+                if (initCalled && !shutdownCalled)
+                {
+                    shutdownCalled = true;
+                    GrpcShutdown();
+                }
+            }
+
+        }
+
+        /// <summary>
+        /// Initializes GRPC C Core library.
+        /// </summary>
+        private static void GrpcInit()
+        {
+            grpc_init();
+            threadPool.Start();
+            // TODO: use proper logging here
+            Console.WriteLine("GRPC initialized.");
+        }
+
+        /// <summary>
+        /// Shutdown GRPC C Core library.
+        /// </summary>
+        private static void GrpcShutdown()
+        {
+            threadPool.Stop();
+            grpc_shutdown();
+
+            // TODO: use proper logging here
+            Console.WriteLine("GRPC shutdown.");
+        }
+
+        internal static GrpcThreadPool ThreadPool
+        {
+            get
+            {
+                return threadPool;
+            }
+        }
+    }
+}
+

src/csharp/GrpcCore/Internal/AsyncCall.cs

@@ -0,0 +1,493 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Runtime.CompilerServices;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// Listener for call events that can be delivered from a completion queue.
+    /// </summary>
+    internal interface ICallEventListener {
+    
+        void OnClientMetadata();
+
+        void OnRead(byte[] payload);
+
+        void OnWriteAccepted(GRPCOpError error);
+
+        void OnFinishAccepted(GRPCOpError error);
+
+        // ignore the status on server
+        void OnFinished(Status status);
+    }
+
+    /// <summary>
+    /// Handle native call lifecycle and provides convenience methods.
+    /// </summary>
+    internal class AsyncCall<TWrite, TRead>: ICallEventListener, IDisposable
+    {
+        readonly Func<TWrite, byte[]> serializer;
+        readonly Func<byte[], TRead> deserializer;
+
+        // TODO: make sure the delegate doesn't get garbage collected while 
+        // native callbacks are in the completion queue.
+        readonly EventCallbackDelegate callbackHandler;
+
+        object myLock = new object();
+        bool disposed;
+        CallSafeHandle call;
+
+        bool started;
+        bool errorOccured;
+
+        bool cancelRequested;
+        bool halfcloseRequested;
+        bool halfclosed;
+        bool doneWithReading;
+        Nullable<Status> finishedStatus;
+
+        TaskCompletionSource<object> writeTcs;
+        TaskCompletionSource<TRead> readTcs;
+        TaskCompletionSource<object> halfcloseTcs = new TaskCompletionSource<object>();
+        TaskCompletionSource<Status> finishedTcs = new TaskCompletionSource<Status>();
+
+        IObserver<TRead> readObserver;
+
+        public AsyncCall(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer)
+        {
+            this.serializer = serializer;
+            this.deserializer = deserializer;
+            this.callbackHandler = HandleEvent;
+        }
+
+        public Task WriteAsync(TWrite msg)
+        {
+            return StartWrite(msg, false).Task;
+        }
+
+        public Task WritesCompletedAsync()
+        {
+            WritesDone();
+            return halfcloseTcs.Task;
+        }
+
+        public Task WriteStatusAsync(Status status)
+        {
+            WriteStatus(status);
+            return halfcloseTcs.Task;
+        }
+
+        public Task<TRead> ReadAsync()
+        {
+            return StartRead().Task;
+        }
+
+        public Task Halfclosed
+        {
+            get
+            {
+                return halfcloseTcs.Task;
+            }
+        }
+
+        public Task<Status> Finished
+        {
+            get
+            {
+                return finishedTcs.Task;
+            }
+        }
+
+        /// <summary>
+        /// Initiates reading to given observer.
+        /// </summary>
+        public void StartReadingToStream(IObserver<TRead> readObserver) {
+            lock (myLock)
+            {
+                CheckStarted();
+                if (this.readObserver != null)
+                {
+                    throw new InvalidOperationException("Already registered an observer.");
+                }
+                this.readObserver = readObserver;
+                StartRead();
+            }
+        }
+
+        public void Initialize(Channel channel, String methodName) {
+            lock (myLock)
+            {
+               this.call = CallSafeHandle.Create(channel.Handle, methodName, channel.Target, Timespec.InfFuture);
+            }
+        }
+
+        public void InitializeServer(CallSafeHandle call)
+        {
+            lock(myLock)
+            {
+                this.call = call;
+            }
+        }
+
+        // Client only
+        public void Start(bool buffered, CompletionQueueSafeHandle cq)
+        {
+            lock (myLock)
+            {
+                if (started)
+                {
+                    throw new InvalidOperationException("Already started.");
+                }
+
+                call.Invoke(cq, buffered, callbackHandler, callbackHandler);
+                started = true;
+            }
+        }
+
+        // Server only
+        public void Accept(CompletionQueueSafeHandle cq)
+        {
+            lock (myLock)
+            {
+                if (started)
+                {
+                    throw new InvalidOperationException("Already started.");
+                }
+
+                call.ServerAccept(cq, callbackHandler);
+                call.ServerEndInitialMetadata(0);
+                started = true;
+            }
+        }
+
+        public TaskCompletionSource<object> StartWrite(TWrite msg, bool buffered)
+        {
+            lock (myLock)
+            {
+                CheckStarted();
+                CheckNotFinished();
+                CheckNoError();
+                CheckCancelNotRequested();
+               
+                if (halfcloseRequested || halfclosed)
+                {
+                    throw new InvalidOperationException("Already halfclosed.");
+                }
+
+                if (writeTcs != null)
+                {
+                    throw new InvalidOperationException("Only one write can be pending at a time");
+                }
+
+                // TODO: wrap serialization...
+                byte[] payload = serializer(msg);
+               
+                call.StartWrite(payload, buffered, callbackHandler);
+                writeTcs = new TaskCompletionSource<object>();
+                return writeTcs;
+            }
+        }
+
+        // client only
+        public void WritesDone()
+        {
+            lock (myLock)
+            {
+                CheckStarted();
+                CheckNotFinished();
+                CheckNoError();
+                CheckCancelNotRequested();
+
+                if (halfcloseRequested || halfclosed)
+                {
+                    throw new InvalidOperationException("Already halfclosed.");
+                }
+
+                call.WritesDone(callbackHandler);
+                halfcloseRequested = true;
+            }
+        }
+
+        // server only
+        public void WriteStatus(Status status)
+        {
+            lock (myLock)
+            {
+                CheckStarted();
+                CheckNotFinished();
+                CheckNoError();
+                CheckCancelNotRequested();
+
+                if (halfcloseRequested || halfclosed)
+                {
+                    throw new InvalidOperationException("Already halfclosed.");
+                }
+
+                call.StartWriteStatus(status, callbackHandler);
+                halfcloseRequested = true;
+            }
+        }
+
+        public TaskCompletionSource<TRead> StartRead()
+        {
+            lock (myLock)
+            {
+                CheckStarted();
+                CheckNotFinished();
+                CheckNoError();
+
+                // TODO: add check for not cancelled?
+
+                if (doneWithReading)
+                {
+                    throw new InvalidOperationException("Already read the last message.");
+                }
+
+                if (readTcs != null)
+                {
+                    throw new InvalidOperationException("Only one read can be pending at a time");
+                }
+
+                call.StartRead(callbackHandler);
+
+                readTcs = new TaskCompletionSource<TRead>();
+                return readTcs;
+            }
+        }
+
+        public void Cancel()
+        {
+            lock (myLock)
+            {
+                CheckStarted();
+                CheckNotFinished();
+
+                cancelRequested = true;
+            }
+            // grpc_call_cancel is threadsafe
+            call.Cancel();
+        }
+
+        public void CancelWithStatus(Status status)
+        {
+            lock (myLock)
+            {
+                CheckStarted();
+                CheckNotFinished();
+
+                cancelRequested = true;
+            }
+            // grpc_call_cancel_with_status is threadsafe
+            call.CancelWithStatus(status);
+        }
+       
+        public void OnClientMetadata()
+        {
+            // TODO: implement....
+        }
+
+        public void OnRead(byte[] payload)
+        {
+            TaskCompletionSource<TRead> oldTcs = null;
+            IObserver<TRead> observer = null;
+            lock (myLock)
+            {
+                oldTcs = readTcs;
+                readTcs = null;
+                if (payload == null)
+                {
+                    doneWithReading = true;
+                }
+                observer = readObserver;
+            }
+
+            // TODO: wrap deserialization...
+            TRead msg = payload != null ? deserializer(payload) : default(TRead);
+
+            oldTcs.SetResult(msg);
+
+            // TODO: make sure we deliver reads in the right order.
+
+            if (observer != null)
+            {
+                if (payload != null)
+                {
+                    // TODO: wrap to handle exceptions
+                    observer.OnNext(msg);
+
+                    // start a new read
+                    StartRead();
+                }
+                else
+                {
+                    // TODO: wrap to handle exceptions;
+                    observer.OnCompleted();
+                }
+
+            }
+        }
+
+        public void OnWriteAccepted(GRPCOpError error)
+        {
+            TaskCompletionSource<object> oldTcs = null;
+            lock (myLock)
+            {
+                UpdateErrorOccured(error);
+                oldTcs = writeTcs;
+                writeTcs = null;
+            }
+
+            if (errorOccured)
+            {
+                // TODO: use the right type of exception...
+                oldTcs.SetException(new Exception("Write failed"));
+            }
+            else
+            {
+                // TODO: where does the continuation run?
+                oldTcs.SetResult(null);
+            }
+        }
+
+        public void OnFinishAccepted(GRPCOpError error)
+        {
+            lock (myLock)
+            {
+                UpdateErrorOccured(error);
+                halfclosed = true;
+            }
+
+            if (errorOccured)
+            {
+                halfcloseTcs.SetException(new Exception("Halfclose failed"));
+
+            }
+            else
+            {
+                halfcloseTcs.SetResult(null);
+            }
+
+        }
+
+        public void OnFinished(Status status)
+        {
+            lock (myLock)
+            {
+                finishedStatus = status;
+
+                DisposeResourcesIfNeeded();
+            }
+            finishedTcs.SetResult(status);
+
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!disposed)
+            {
+                if (disposing)
+                {
+                    if (call != null)
+                    {
+                        call.Dispose();
+                    }
+                } 
+                disposed = true;
+            }
+        }
+
+        private void UpdateErrorOccured(GRPCOpError error)
+        {
+            if (error == GRPCOpError.GRPC_OP_ERROR)
+            {
+                errorOccured = true;
+            }
+        }
+
+        private void CheckStarted()
+        {
+            if (!started)
+            {
+                throw new InvalidOperationException("Call not started");
+            }
+        }
+
+        private void CheckNoError()
+        {
+            if (errorOccured)
+            {
+                throw new InvalidOperationException("Error occured when processing call.");
+            }
+        }
+
+        private void CheckNotFinished()
+        {
+            if (finishedStatus.HasValue)
+            {
+                throw new InvalidOperationException("Already finished.");
+            }
+        }
+
+        private void CheckCancelNotRequested()
+        {
+            if (cancelRequested)
+            {
+                throw new InvalidOperationException("Cancel has been requested.");
+            }
+        }
+
+        private void DisposeResourcesIfNeeded()
+        {
+            if (call != null && started && finishedStatus.HasValue)
+            {
+                // TODO: should we also wait for all the pending events to finish?
+
+                call.Dispose();
+            }
+        }
+
+        private void HandleEvent(IntPtr eventPtr) {
+            try {
+                var ev = new EventSafeHandleNotOwned(eventPtr);
+                switch (ev.GetCompletionType())
+                {
+                case GRPCCompletionType.GRPC_CLIENT_METADATA_READ:
+                    OnClientMetadata();
+                    break;
+
+                case GRPCCompletionType.GRPC_READ:
+                    byte[] payload = ev.GetReadData();
+                    OnRead(payload);
+                    break;
+
+                case GRPCCompletionType.GRPC_WRITE_ACCEPTED:
+                    OnWriteAccepted(ev.GetWriteAccepted());
+                    break;
+
+                case GRPCCompletionType.GRPC_FINISH_ACCEPTED:
+                    OnFinishAccepted(ev.GetFinishAccepted());
+                    break;
+
+                case GRPCCompletionType.GRPC_FINISHED:
+                    OnFinished(ev.GetFinished());
+                    break;
+
+                default:
+                    throw new ArgumentException("Unexpected completion type");
+                }
+            } catch(Exception e) {
+                Console.WriteLine("Caught exception in a native handler: " + e);
+            }
+        }
+    }
+}

src/csharp/GrpcCore/Internal/CallSafeHandle.cs

@@ -0,0 +1,182 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Diagnostics;
+using Google.GRPC.Core;
+
+namespace Google.GRPC.Core.Internal
+{
+    // TODO: we need to make sure that the delegates are not collected before invoked.
+    internal delegate void EventCallbackDelegate(IntPtr eventPtr);
+
+    /// <summary>
+    /// grpc_call from <grpc/grpc.h>
+    /// </summary>
+	internal class CallSafeHandle : SafeHandleZeroIsInvalid
+	{
+        const UInt32 GRPC_WRITE_BUFFER_HINT = 1;
+
+        [DllImport("libgrpc.so")]
+        static extern CallSafeHandle grpc_channel_create_call_old(ChannelSafeHandle channel, string method, string host, Timespec deadline);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_add_metadata(CallSafeHandle call, IntPtr metadata, UInt32 flags);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_invoke_old(CallSafeHandle call, CompletionQueueSafeHandle cq, IntPtr metadataReadTag, IntPtr finishedTag, UInt32 flags);
+
+        [DllImport("libgrpc.so", EntryPoint = "grpc_call_invoke_old")]
+        static extern GRPCCallError grpc_call_invoke_old_CALLBACK(CallSafeHandle call, CompletionQueueSafeHandle cq,
+                                                              [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate metadataReadCallback, 
+                                                              [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate finishedCallback, 
+                                                              UInt32 flags);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_server_accept_old(CallSafeHandle call, CompletionQueueSafeHandle completionQueue, IntPtr finishedTag);
+
+        [DllImport("libgrpc.so", EntryPoint = "grpc_call_server_accept_old")]
+        static extern GRPCCallError grpc_call_server_accept_old_CALLBACK(CallSafeHandle call, CompletionQueueSafeHandle completionQueue, [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate finishedCallback);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_server_end_initial_metadata_old(CallSafeHandle call, UInt32 flags);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_cancel(CallSafeHandle call);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_start_write_status_old(CallSafeHandle call, StatusCode statusCode, string statusMessage, IntPtr tag);
+
+        [DllImport("libgrpc.so", EntryPoint = "grpc_call_start_write_status_old")]
+        static extern GRPCCallError grpc_call_start_write_status_old_CALLBACK(CallSafeHandle call, StatusCode statusCode, string statusMessage, [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate callback);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_writes_done_old(CallSafeHandle call, IntPtr tag);
+
+        [DllImport("libgrpc.so", EntryPoint = "grpc_call_writes_done_old")]
+        static extern GRPCCallError grpc_call_writes_done_old_CALLBACK(CallSafeHandle call, [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate callback);
+
+        [DllImport("libgrpc.so")]
+        static extern GRPCCallError grpc_call_start_read_old(CallSafeHandle call, IntPtr tag);
+
+        [DllImport("libgrpc.so", EntryPoint = "grpc_call_start_read_old")]
+        static extern GRPCCallError grpc_call_start_read_old_CALLBACK(CallSafeHandle call, [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate callback);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern void grpc_call_start_write_from_copied_buffer(CallSafeHandle call,
+                                                                    byte[] buffer, UIntPtr length,
+                                                                    IntPtr tag, UInt32 flags);
+
+        [DllImport("libgrpc_csharp_ext.so", EntryPoint = "grpc_call_start_write_from_copied_buffer")]
+        static extern void grpc_call_start_write_from_copied_buffer_CALLBACK(CallSafeHandle call,
+                                                                             byte[] buffer, UIntPtr length,
+                                                                             [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate callback,
+                                                                             UInt32 flags);
+
+		[DllImport("libgrpc.so")]
+		static extern void grpc_call_destroy(IntPtr call);
+
+        private CallSafeHandle()
+        {
+        }
+
+        /// <summary>
+        /// Creates a client call.
+        /// </summary>
+        public static CallSafeHandle Create(ChannelSafeHandle channel, string method, string host, Timespec deadline)
+        {
+            return grpc_channel_create_call_old(channel, method, host, deadline);
+        }
+
+        public void Invoke(CompletionQueueSafeHandle cq, IntPtr metadataReadTag, IntPtr finishedTag, bool buffered)
+        {   
+            AssertCallOk(grpc_call_invoke_old(this, cq, metadataReadTag, finishedTag, GetFlags(buffered)));
+        }
+
+        public void Invoke(CompletionQueueSafeHandle cq, bool buffered, EventCallbackDelegate metadataReadCallback, EventCallbackDelegate finishedCallback)
+        {   
+            AssertCallOk(grpc_call_invoke_old_CALLBACK(this, cq, metadataReadCallback, finishedCallback, GetFlags(buffered)));
+        }
+
+        public void ServerAccept(CompletionQueueSafeHandle cq, IntPtr finishedTag)
+        {
+            AssertCallOk(grpc_call_server_accept_old(this, cq, finishedTag));
+        }
+
+        public void ServerAccept(CompletionQueueSafeHandle cq, EventCallbackDelegate callback)
+        {
+            AssertCallOk(grpc_call_server_accept_old_CALLBACK(this, cq, callback));
+        }
+
+        public void ServerEndInitialMetadata(UInt32 flags)
+        {
+            AssertCallOk(grpc_call_server_end_initial_metadata_old(this, flags));
+        }
+
+        public void StartWrite(byte[] payload, IntPtr tag, bool buffered)
+        {
+            grpc_call_start_write_from_copied_buffer(this, payload, new UIntPtr((ulong) payload.Length), tag, GetFlags(buffered));
+        }
+
+        public void StartWrite(byte[] payload, bool buffered, EventCallbackDelegate callback)
+        {
+            grpc_call_start_write_from_copied_buffer_CALLBACK(this, payload, new UIntPtr((ulong) payload.Length), callback, GetFlags(buffered));
+        }
+
+        public void StartWriteStatus(Status status, IntPtr tag)
+        {
+            AssertCallOk(grpc_call_start_write_status_old(this, status.StatusCode, status.Detail, tag));
+        }
+
+        public void StartWriteStatus(Status status, EventCallbackDelegate callback)
+        {
+            AssertCallOk(grpc_call_start_write_status_old_CALLBACK(this, status.StatusCode, status.Detail, callback));
+        }
+
+        public void WritesDone(IntPtr tag)
+        {
+            AssertCallOk(grpc_call_writes_done_old(this, tag));
+        }
+
+        public void WritesDone(EventCallbackDelegate callback)
+        {
+            AssertCallOk(grpc_call_writes_done_old_CALLBACK(this, callback));
+        }
+
+        public void StartRead(IntPtr tag)
+        {
+            AssertCallOk(grpc_call_start_read_old(this, tag));
+        }
+
+        public void StartRead(EventCallbackDelegate callback)
+        {
+            AssertCallOk(grpc_call_start_read_old_CALLBACK(this, callback));
+        }
+
+        public void Cancel()
+        {
+            AssertCallOk(grpc_call_cancel(this));
+        }
+
+        public void CancelWithStatus(Status status)
+        {
+            AssertCallOk(grpc_call_cancel_with_status(this, status.StatusCode, status.Detail));
+        }
+
+		protected override bool ReleaseHandle()
+		{
+			grpc_call_destroy(handle);
+			return true;
+		}
+
+        private static void AssertCallOk(GRPCCallError callError)
+        {
+            Trace.Assert(callError == GRPCCallError.GRPC_CALL_OK, "Status not GRPC_CALL_OK");
+        }
+
+        private static UInt32 GetFlags(bool buffered) {
+            return buffered ? 0 : GRPC_WRITE_BUFFER_HINT;
+        }
+	}
+}

src/csharp/GrpcCore/Internal/ChannelSafeHandle.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// grpc_channel from <grpc/grpc.h>
+    /// </summary>
+	internal class ChannelSafeHandle : SafeHandleZeroIsInvalid
+	{
+        [DllImport("libgrpc.so")]
+        static extern ChannelSafeHandle grpc_channel_create(string target, IntPtr channelArgs);
+
+		[DllImport("libgrpc.so")]
+		static extern void grpc_channel_destroy(IntPtr channel);
+
+        private ChannelSafeHandle()
+        {
+        }
+
+        public static ChannelSafeHandle Create(string target, IntPtr channelArgs)
+        {
+            return grpc_channel_create(target, channelArgs);
+        }
+
+		protected override bool ReleaseHandle()
+		{
+			grpc_channel_destroy(handle);
+			return true;
+		}
+	}
+}

src/csharp/GrpcCore/Internal/CompletionQueueSafeHandle.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// grpc_completion_queue from <grpc/grpc.h>
+    /// </summary>
+	internal class CompletionQueueSafeHandle : SafeHandleZeroIsInvalid
+	{
+        [DllImport("libgrpc.so")]
+        static extern CompletionQueueSafeHandle grpc_completion_queue_create();
+
+        [DllImport("libgrpc.so")]
+        static extern EventSafeHandle grpc_completion_queue_pluck(CompletionQueueSafeHandle cq, IntPtr tag, Timespec deadline);
+
+        [DllImport("libgrpc.so")]
+        static extern EventSafeHandle grpc_completion_queue_next(CompletionQueueSafeHandle cq, Timespec deadline);
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_completion_queue_shutdown(CompletionQueueSafeHandle cq);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCCompletionType grpc_completion_queue_next_with_callback(CompletionQueueSafeHandle cq);
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_completion_queue_destroy(IntPtr cq);
+
+        private CompletionQueueSafeHandle()
+        {
+        }
+
+        public static CompletionQueueSafeHandle Create()
+        {
+            return grpc_completion_queue_create();
+        }
+
+        public EventSafeHandle Next(Timespec deadline)
+        {
+            return grpc_completion_queue_next(this, deadline);
+        }
+
+        public GRPCCompletionType NextWithCallback()
+        {
+            return grpc_completion_queue_next_with_callback(this);
+        }
+
+        public EventSafeHandle Pluck(IntPtr tag, Timespec deadline)
+        {
+            return grpc_completion_queue_pluck(this, tag, deadline);
+        }
+
+        public void Shutdown()
+        {
+            grpc_completion_queue_shutdown(this);
+        }
+
+		protected override bool ReleaseHandle()
+        {
+            grpc_completion_queue_destroy(handle);
+			return true;
+		}
+	}
+}
+

src/csharp/GrpcCore/Internal/Enums.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// from grpc/grpc.h
+    /// </summary>
+    internal enum GRPCCallError
+    {
+        /* everything went ok */
+        GRPC_CALL_OK = 0,
+        /* something failed, we don't know what */
+        GRPC_CALL_ERROR,
+        /* this method is not available on the server */
+        GRPC_CALL_ERROR_NOT_ON_SERVER,
+        /* this method is not available on the client */
+        GRPC_CALL_ERROR_NOT_ON_CLIENT,
+        /* this method must be called before server_accept */
+        GRPC_CALL_ERROR_ALREADY_ACCEPTED,
+        /* this method must be called before invoke */
+        GRPC_CALL_ERROR_ALREADY_INVOKED,
+        /* this method must be called after invoke */
+        GRPC_CALL_ERROR_NOT_INVOKED,
+        /* this call is already finished
+     (writes_done or write_status has already been called) */
+        GRPC_CALL_ERROR_ALREADY_FINISHED,
+        /* there is already an outstanding read/write operation on the call */
+        GRPC_CALL_ERROR_TOO_MANY_OPERATIONS,
+        /* the flags value was illegal for this call */
+        GRPC_CALL_ERROR_INVALID_FLAGS
+    }
+
+    /// <summary>
+    /// grpc_completion_type from grpc/grpc.h
+    /// </summary>
+    internal enum GRPCCompletionType
+    {
+        GRPC_QUEUE_SHUTDOWN,
+        /* Shutting down */
+        GRPC_READ,
+        /* A read has completed */
+        GRPC_INVOKE_ACCEPTED,
+        /* An invoke call has been accepted by flow
+                                control */
+        GRPC_WRITE_ACCEPTED,
+        /* A write has been accepted by
+                                flow control */
+        GRPC_FINISH_ACCEPTED,
+        /* writes_done or write_status has been accepted */
+        GRPC_CLIENT_METADATA_READ,
+        /* The metadata array sent by server received at
+                                client */
+        GRPC_FINISHED,
+        /* An RPC has finished. The event contains status.
+                                On the server this will be OK or Cancelled. */
+        GRPC_SERVER_RPC_NEW,
+        /* A new RPC has arrived at the server */
+        GRPC_COMPLETION_DO_NOT_USE
+        /* must be last, forces users to include
+                                a default: case */
+    }
+
+    /// <summary>
+    /// grpc_op_error from grpc/grpc.h
+    /// </summary>
+    internal enum GRPCOpError
+    {
+        /* everything went ok */
+        GRPC_OP_OK = 0,
+        /* something failed, we don't know what */
+        GRPC_OP_ERROR
+    }
+}
+

src/csharp/GrpcCore/Internal/Event.cs

@@ -0,0 +1,191 @@
+using System;
+using System.Runtime.InteropServices;
+using Google.GRPC.Core;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// grpc_event from grpc/grpc.h
+    /// </summary>
+    internal class EventSafeHandle : SafeHandleZeroIsInvalid
+    {
+        [DllImport("libgrpc.so")]
+        static extern void grpc_event_finish(IntPtr ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCCompletionType grpc_event_type(EventSafeHandle ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern CallSafeHandle grpc_event_call(EventSafeHandle ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCOpError grpc_event_write_accepted(EventSafeHandle ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCOpError grpc_event_finish_accepted(EventSafeHandle ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern StatusCode grpc_event_finished_status(EventSafeHandle ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern IntPtr grpc_event_finished_details(EventSafeHandle ev);  // returns const char*
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern IntPtr grpc_event_read_length(EventSafeHandle ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern void grpc_event_read_copy_to_buffer(EventSafeHandle ev, byte[] buffer, UIntPtr bufferLen);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern IntPtr grpc_event_server_rpc_new_method(EventSafeHandle ev); // returns const char*
+
+        public GRPCCompletionType GetCompletionType()
+        {
+            return grpc_event_type(this);
+        }
+
+        public GRPCOpError GetWriteAccepted()
+        {
+            return grpc_event_write_accepted(this);
+        }
+
+        public GRPCOpError GetFinishAccepted()
+        {
+            return grpc_event_finish_accepted(this);
+        }
+
+        public Status GetFinished()
+        {
+            // TODO: can the native method return string directly?
+            string details = Marshal.PtrToStringAnsi(grpc_event_finished_details(this));
+            return new Status(grpc_event_finished_status(this), details);
+        }
+
+        public byte[] GetReadData()
+        {
+            IntPtr len = grpc_event_read_length(this);
+            if (len == new IntPtr(-1))
+            {
+                return null;
+            }
+            byte[] data = new byte[(int) len];
+            grpc_event_read_copy_to_buffer(this, data, new UIntPtr((ulong)data.Length));
+            return data;
+        }
+
+        public CallSafeHandle GetCall() {
+            return grpc_event_call(this);
+        }
+
+        public string GetServerRpcNewMethod() {
+            // TODO: can the native method return string directly?
+            return Marshal.PtrToStringAnsi(grpc_event_server_rpc_new_method(this));
+        }
+
+        //TODO: client_metadata_read event type
+
+        protected override bool ReleaseHandle()
+        {
+            grpc_event_finish(handle);
+            return true;
+        }
+    }
+
+    // TODO: this is basically c&p of EventSafeHandle. Unify!
+    /// <summary>
+    /// Not owned version of 
+    /// grpc_event from grpc/grpc.h
+    /// </summary>
+    internal class EventSafeHandleNotOwned : SafeHandleZeroIsInvalid
+    {
+        [DllImport("libgrpc.so")]
+        static extern void grpc_event_finish(IntPtr ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCCompletionType grpc_event_type(EventSafeHandleNotOwned ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern CallSafeHandle grpc_event_call(EventSafeHandleNotOwned ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCOpError grpc_event_write_accepted(EventSafeHandleNotOwned ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern GRPCOpError grpc_event_finish_accepted(EventSafeHandleNotOwned ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern StatusCode grpc_event_finished_status(EventSafeHandleNotOwned ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern IntPtr grpc_event_finished_details(EventSafeHandleNotOwned ev);  // returns const char*
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern IntPtr grpc_event_read_length(EventSafeHandleNotOwned ev);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern void grpc_event_read_copy_to_buffer(EventSafeHandleNotOwned ev, byte[] buffer, UIntPtr bufferLen);
+
+        [DllImport("libgrpc_csharp_ext.so")]
+        static extern IntPtr grpc_event_server_rpc_new_method(EventSafeHandleNotOwned ev); // returns const char*
+
+        public EventSafeHandleNotOwned() : base(false)
+        {
+        }
+
+        public EventSafeHandleNotOwned(IntPtr handle) : base(false)
+        {
+            SetHandle(handle);
+        }
+
+        public GRPCCompletionType GetCompletionType()
+        {
+            return grpc_event_type(this);
+        }
+
+        public GRPCOpError GetWriteAccepted()
+        {
+            return grpc_event_write_accepted(this);
+        }
+
+        public GRPCOpError GetFinishAccepted()
+        {
+            return grpc_event_finish_accepted(this);
+        }
+
+        public Status GetFinished()
+        {
+            // TODO: can the native method return string directly?
+            string details = Marshal.PtrToStringAnsi(grpc_event_finished_details(this));
+            return new Status(grpc_event_finished_status(this), details);
+        }
+
+        public byte[] GetReadData()
+        {
+            IntPtr len = grpc_event_read_length(this);
+            if (len == new IntPtr(-1))
+            {
+                return null;
+            }
+            byte[] data = new byte[(int) len];
+            grpc_event_read_copy_to_buffer(this, data, new UIntPtr((ulong)data.Length));
+            return data;
+        }
+
+        public CallSafeHandle GetCall() {
+            return grpc_event_call(this);
+        }
+
+        public string GetServerRpcNewMethod() {
+            // TODO: can the native method return string directly?
+            return Marshal.PtrToStringAnsi(grpc_event_server_rpc_new_method(this));
+        }
+
+        //TODO: client_metadata_read event type
+
+        protected override bool ReleaseHandle()
+        {
+            grpc_event_finish(handle);
+            return true;
+        }
+    }
+}

src/csharp/GrpcCore/Internal/GrpcThreadPool.cs

@@ -0,0 +1,129 @@
+using System;
+using Google.GRPC.Core.Internal;
+using System.Runtime.InteropServices;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Collections.Generic;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// Pool of threads polling on the same completion queue.
+    /// </summary>
+    internal class GrpcThreadPool
+    {
+        readonly object myLock = new object();
+        readonly List<Thread> threads = new List<Thread>();
+        readonly int poolSize;
+        readonly Action<EventSafeHandle> eventHandler;
+
+        CompletionQueueSafeHandle cq;
+
+        public GrpcThreadPool(int poolSize) {
+            this.poolSize = poolSize;
+        }
+
+        internal GrpcThreadPool(int poolSize, Action<EventSafeHandle> eventHandler) {
+            this.poolSize = poolSize;
+            this.eventHandler = eventHandler;
+        }
+
+        public void Start() {
+
+            lock (myLock)
+            {
+                if (cq != null)
+                {
+                    throw new InvalidOperationException("Already started.");
+                }
+
+                cq = CompletionQueueSafeHandle.Create();
+
+                for (int i = 0; i < poolSize; i++)
+                {
+                    threads.Add(CreateAndStartThread(i));
+                }
+            }
+        }
+
+        public void Stop() {
+
+            lock (myLock)
+            {
+                cq.Shutdown();
+
+                Console.WriteLine("Waiting for GPRC threads to finish.");
+                foreach (var thread in threads)
+                {
+                    thread.Join();
+                }
+
+                cq.Dispose();
+
+            }
+        }
+
+        internal CompletionQueueSafeHandle CompletionQueue
+        {
+            get
+            {
+                return cq;
+            }
+        }
+
+        private Thread CreateAndStartThread(int i) {
+            Action body;
+            if (eventHandler != null)
+            {
+                body = ThreadBodyWithHandler;
+            }
+            else
+            {
+                body = ThreadBodyNoHandler;
+            }
+            var thread = new Thread(new ThreadStart(body));
+            thread.IsBackground = false;
+            thread.Start();
+            if (eventHandler != null)
+            {
+                thread.Name = "grpc_server_newrpc " + i;
+            }
+            else
+            {
+                thread.Name = "grpc " + i;
+            }
+            return thread;
+        }
+
+        /// <summary>
+        /// Body of the polling thread.
+        /// </summary>
+        private void ThreadBodyNoHandler()
+        {
+            GRPCCompletionType completionType;
+            do
+            {
+                completionType = cq.NextWithCallback();
+            } while(completionType != GRPCCompletionType.GRPC_QUEUE_SHUTDOWN);
+            Console.WriteLine("Completion queue has shutdown successfully, thread " + Thread.CurrentThread.Name + " exiting.");
+        }
+
+        /// <summary>
+        /// Body of the polling thread.
+        /// </summary>
+        private void ThreadBodyWithHandler()
+        {
+            GRPCCompletionType completionType;
+            do
+            {
+                using (EventSafeHandle ev = cq.Next(Timespec.InfFuture)) {
+                    completionType = ev.GetCompletionType();
+                    eventHandler(ev);
+                }
+            } while(completionType != GRPCCompletionType.GRPC_QUEUE_SHUTDOWN);
+            Console.WriteLine("Completion queue has shutdown successfully, thread " + Thread.CurrentThread.Name + " exiting.");
+        }
+    }
+
+}
+

src/csharp/GrpcCore/Internal/SafeHandleZeroIsInvalid.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Runtime.InteropServices;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// Safe handle to wrap native objects.
+    /// </summary>
+    internal abstract class SafeHandleZeroIsInvalid : SafeHandle
+    {
+        public SafeHandleZeroIsInvalid() : base(IntPtr.Zero, true)
+        {
+        }
+
+        public SafeHandleZeroIsInvalid(bool ownsHandle) : base(IntPtr.Zero, ownsHandle)
+        {
+        }
+
+        public override bool IsInvalid
+        {
+            get
+            {
+                return handle == IntPtr.Zero;
+            }
+        }
+    }
+}
+

src/csharp/GrpcCore/Internal/ServerSafeHandle.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Diagnostics;
+using System.Collections.Concurrent;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// grpc_server from grpc/grpc.h
+    /// </summary>
+    internal sealed class ServerSafeHandle : SafeHandleZeroIsInvalid
+    {
+        [DllImport("libgrpc.so", EntryPoint = "grpc_server_request_call_old")]
+        static extern GRPCCallError grpc_server_request_call_old_CALLBACK(ServerSafeHandle server, [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate callback);
+
+        [DllImport("libgrpc.so")]
+        static extern ServerSafeHandle grpc_server_create(CompletionQueueSafeHandle cq, IntPtr args);
+
+        // TODO: check int representation size
+        [DllImport("libgrpc.so")]
+        static extern int grpc_server_add_http2_port(ServerSafeHandle server, string addr);
+
+        // TODO: check int representation size
+        [DllImport("libgrpc.so")]
+        static extern int grpc_server_add_secure_http2_port(ServerSafeHandle server, string addr);
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_server_start(ServerSafeHandle server);
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_server_shutdown(ServerSafeHandle server);
+
+        [DllImport("libgrpc.so", EntryPoint = "grpc_server_shutdown_and_notify")]
+        static extern void grpc_server_shutdown_and_notify_CALLBACK(ServerSafeHandle server, [MarshalAs(UnmanagedType.FunctionPtr)] EventCallbackDelegate callback);
+
+        [DllImport("libgrpc.so")]
+        static extern void grpc_server_destroy(IntPtr server);
+
+        private ServerSafeHandle()
+        {
+        }
+
+        public static ServerSafeHandle NewServer(CompletionQueueSafeHandle cq, IntPtr args)
+        {
+            // TODO: also grpc_secure_server_create...
+            return grpc_server_create(cq, args);
+        }
+
+        public int AddPort(string addr)
+        {
+            // TODO: also grpc_server_add_secure_http2_port...
+            return grpc_server_add_http2_port(this, addr);
+        }
+
+        public void Start()
+        {
+            grpc_server_start(this);
+        }
+
+        public void Shutdown()
+        {
+            grpc_server_shutdown(this);
+        }
+
+        public void ShutdownAndNotify(EventCallbackDelegate callback)
+        {
+            grpc_server_shutdown_and_notify_CALLBACK(this, callback);
+        }
+
+        public GRPCCallError RequestCall(EventCallbackDelegate callback)
+        {
+            return grpc_server_request_call_old_CALLBACK(this, callback);
+        }
+
+        protected override bool ReleaseHandle()
+        {
+            grpc_server_destroy(handle);
+            return true;
+        }
+    }
+}

src/csharp/GrpcCore/Internal/ServerWritingObserver.cs

@@ -0,0 +1,38 @@
+using System;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core.Internal
+{
+    /// <summary>
+    /// Observer that writes all arriving messages to a call abstraction (in blocking fashion)
+    /// and then halfcloses the call. Used for server-side call handling.
+    /// </summary>
+    internal class ServerWritingObserver<TWrite, TRead> : IObserver<TWrite>
+	{
+        readonly AsyncCall<TWrite, TRead> call;
+
+        public ServerWritingObserver(AsyncCall<TWrite, TRead> call)
+		{
+            this.call = call;
+		}
+
+		public void OnCompleted()
+		{
+            // TODO: how bad is the Wait here?
+            call.WriteStatusAsync(new Status(StatusCode.GRPC_STATUS_OK, "")).Wait();
+		}
+
+		public void OnError(Exception error)
+		{
+            // TODO: handle this...
+			throw new InvalidOperationException("This should never be called.");
+		}
+
+		public void OnNext(TWrite value)
+		{
+            // TODO: how bad is the Wait here?
+            call.WriteAsync(value).Wait();
+		}
+	}
+}
+

src/csharp/GrpcCore/Internal/StreamingInputObserver.cs

@@ -0,0 +1,33 @@
+using System;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core.Internal
+{
+    internal class StreamingInputObserver<TWrite, TRead> : IObserver<TWrite>
+	{
+        readonly AsyncCall<TWrite, TRead> call;
+
+        public StreamingInputObserver(AsyncCall<TWrite, TRead> call)
+		{
+            this.call = call;
+		}
+
+		public void OnCompleted()
+		{
+            // TODO: how bad is the Wait here?
+            call.WritesCompletedAsync().Wait();
+		}
+
+		public void OnError(Exception error)
+		{
+			throw new InvalidOperationException("This should never be called.");
+		}
+
+		public void OnNext(TWrite value)
+		{
+            // TODO: how bad is the Wait here?
+            call.WriteAsync(value).Wait();
+		}
+	}
+}
+

src/csharp/GrpcCore/Internal/Timespec.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+namespace Google.GRPC.Core.Internal
+{
+	/// <summary>
+	/// gpr_timespec from grpc/support/time.h
+	/// </summary>
+	[StructLayout(LayoutKind.Sequential)]
+	internal struct Timespec
+	{
+        const int nanosPerSecond = 1000 * 1000 * 1000;
+        const int nanosPerTick = 100;
+
+        [DllImport("libgpr.so")]
+        static extern Timespec gpr_now();
+
+		// TODO: this only works on 64bit linux, can we autoselect the right size of ints?
+		// perhaps using IntPtr would work.
+		public System.Int64 tv_sec;
+		public System.Int64 tv_nsec;
+
+		/// <summary>
+		/// Timespec a long time in the future.
+		/// </summary>
+		public static Timespec InfFuture
+		{
+			get
+			{
+				// TODO: set correct value based on the length of the struct
+				return new Timespec { tv_sec = Int32.MaxValue, tv_nsec = 0 };
+			}
+		}
+
+        public static Timespec Now
+        {
+            get
+            {
+                return gpr_now();
+            }
+        }
+
+        /// <summary>
+        /// Creates a GPR deadline from current instant and given timeout.
+        /// </summary>
+        /// <returns>The from timeout.</returns>
+        public static Timespec DeadlineFromTimeout(TimeSpan timeout) {
+            if (timeout == Timeout.InfiniteTimeSpan)
+            {
+                return Timespec.InfFuture;
+            }
+            return Timespec.Now.Add(timeout);
+        }
+
+        public Timespec Add(TimeSpan timeSpan) {
+            long nanos = tv_nsec + (timeSpan.Ticks % TimeSpan.TicksPerSecond) * nanosPerTick;
+            long overflow_sec = (nanos > nanosPerSecond) ? 1 : 0;
+
+            Timespec result;
+            result.tv_nsec = nanos % nanosPerSecond;
+            result.tv_sec = tv_sec + (timeSpan.Ticks / TimeSpan.TicksPerSecond) + overflow_sec; 
+            return result;
+        }
+	}
+}
+

src/csharp/GrpcCore/Marshaller.cs

@@ -0,0 +1,54 @@
+using System;
+
+namespace Google.GRPC.Core
+{
+    /// <summary>
+    /// For serializing and deserializing messages.
+    /// </summary>
+    public struct Marshaller<T>
+    {
+        readonly Func<T,byte[]> serializer;
+        readonly Func<byte[],T> deserializer;
+
+        public Marshaller(Func<T, byte[]> serializer, Func<byte[], T> deserializer)
+        {
+            this.serializer = serializer;
+            this.deserializer = deserializer;
+        }
+
+        public Func<T, byte[]> Serializer
+        {
+            get
+            {
+                return this.serializer;
+            }
+        }
+
+        public Func<byte[], T> Deserializer
+        {
+            get
+            {
+                return this.deserializer;
+            }
+        }
+    }
+
+    public static class Marshallers {
+
+        public static Marshaller<T> Create<T>(Func<T,byte[]> serializer, Func<byte[],T> deserializer)
+        {
+            return new Marshaller<T>(serializer, deserializer);
+        }
+
+        public static Marshaller<string> StringMarshaller
+        {
+            get
+            {
+                return new Marshaller<string>(System.Text.Encoding.UTF8.GetBytes, 
+                                              System.Text.Encoding.UTF8.GetString);
+            }
+        }
+
+    }
+}
+

src/csharp/GrpcCore/Method.cs

@@ -0,0 +1,64 @@
+using System;
+
+namespace Google.GRPC.Core
+{
+    public enum MethodType
+    {
+        Unary,
+        ClientStreaming,
+        ServerStreaming,
+        DuplexStreaming
+    }
+
+    /// <summary>
+    /// A description of a service method.
+    /// </summary>
+    public class Method<TRequest, TResponse>
+    {
+        readonly MethodType type;
+        readonly string name;
+        readonly Marshaller<TRequest> requestMarshaller;
+        readonly Marshaller<TResponse> responseMarshaller;
+
+        public Method(MethodType type, string name, Marshaller<TRequest> requestMarshaller, Marshaller<TResponse> responseMarshaller)
+        {
+            this.type = type;
+            this.name = name;
+            this.requestMarshaller = requestMarshaller;
+            this.responseMarshaller = responseMarshaller;
+        }
+
+        public MethodType Type
+        {
+            get
+            {
+                return this.type;
+            }
+        }
+
+        public string Name
+        {
+            get
+            {
+                return this.name;
+            }
+        }
+
+        public Marshaller<TRequest> RequestMarshaller
+        {
+            get
+            {
+                return this.requestMarshaller;
+            }
+        }
+
+        public Marshaller<TResponse> ResponseMarshaller
+        {
+            get
+            {
+                return this.responseMarshaller;
+            }
+        }
+    }
+}
+

src/csharp/GrpcCore/Properties/AssemblyInfo.cs

@@ -0,0 +1,24 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+// Information about this assembly is defined by the following attributes. 
+// Change them to the values specific to your project.
+[assembly: AssemblyTitle ("GrpcCore")]
+[assembly: AssemblyDescription ("")]
+[assembly: AssemblyConfiguration ("")]
+[assembly: AssemblyCompany ("")]
+[assembly: AssemblyProduct ("")]
+[assembly: AssemblyCopyright ("jtattermusch")]
+[assembly: AssemblyTrademark ("")]
+[assembly: AssemblyCulture ("")]
+// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
+// The form "{Major}.{Minor}.*" will automatically update the build and revision,
+// and "{Major}.{Minor}.{Build}.*" will update just the revision.
+[assembly: AssemblyVersion ("1.0.*")]
+// The following attributes are used to specify the signing key for the assembly, 
+// if desired. See the Mono documentation for more information about signing.
+//[assembly: AssemblyDelaySign(false)]
+//[assembly: AssemblyKeyFile("")]
+
+[assembly: InternalsVisibleTo("GrpcCoreTests")]
+

src/csharp/GrpcCore/RpcException.cs

@@ -0,0 +1,27 @@
+using System;
+
+namespace Google.GRPC.Core
+{
+    public class RpcException : Exception
+    {
+        private readonly Status status;
+
+        public RpcException(Status status)
+        {
+            this.status = status;
+        }
+
+        public RpcException(Status status, string message) : base(message)
+        {
+            this.status = status;
+        }
+
+        public Status Status {
+            get
+            {
+                return status;
+            }
+        }
+    }
+}
+

src/csharp/GrpcCore/Server.cs

@@ -0,0 +1,183 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using Google.GRPC.Core.Internal;
+
+namespace Google.GRPC.Core
+{
+    /// <summary>
+    /// Server is implemented only to be able to do
+    /// in-process testing.
+    /// </summary>
+    public class Server
+    {
+        // TODO: make sure the delegate doesn't get garbage collected while 
+        // native callbacks are in the completion queue.
+        readonly EventCallbackDelegate newRpcHandler;
+        readonly EventCallbackDelegate serverShutdownHandler;
+
+        readonly BlockingCollection<NewRpcInfo> newRpcQueue = new BlockingCollection<NewRpcInfo>();
+        readonly ServerSafeHandle handle;
+
+        readonly Dictionary<string, IServerCallHandler> callHandlers = new Dictionary<string, IServerCallHandler>();
+
+        readonly TaskCompletionSource<object> shutdownTcs = new TaskCompletionSource<object>();
+
+        static Server() {
+            GrpcEnvironment.EnsureInitialized();
+        }
+
+        public Server()
+        {
+            // TODO: what is the tag for server shutdown?
+            this.handle = ServerSafeHandle.NewServer(GetCompletionQueue(), IntPtr.Zero);
+            this.newRpcHandler = HandleNewRpc;
+            this.serverShutdownHandler = HandleServerShutdown;
+        }
+
+        // only call this before Start()
+        public void AddServiceDefinition(ServerServiceDefinition serviceDefinition) {
+            foreach(var entry in serviceDefinition.CallHandlers)
+            {
+                callHandlers.Add(entry.Key, entry.Value);
+            }
+        }
+
+        // only call before Start()
+        public int AddPort(string addr) {
+            return handle.AddPort(addr);
+        }
+
+        public void Start()
+        {
+            handle.Start();
+
+            // TODO: this basically means the server is single threaded....
+            StartHandlingRpcs();
+        }
+
+        /// <summary>
+        /// Requests and handles single RPC call.
+        /// </summary>
+        internal void RunRpc()
+        {
+            AllowOneRpc();
+         
+            try
+            {
+                var rpcInfo = newRpcQueue.Take();
+
+                Console.WriteLine("Server received RPC " + rpcInfo.Method);
+
+                IServerCallHandler callHandler;
+                if (!callHandlers.TryGetValue(rpcInfo.Method, out callHandler))
+                {
+                    callHandler = new NoSuchMethodCallHandler();
+                } 
+                callHandler.StartCall(rpcInfo.Method, rpcInfo.Call, GetCompletionQueue());
+            }
+            catch(Exception e)
+            {
+                Console.WriteLine("Exception while handling RPC: " + e);
+            }
+        }
+
+        /// <summary>
+        /// Requests server shutdown and when there are no more calls being serviced,
+        /// cleans up used resources.
+        /// </summary>
+        /// <returns>The async.</returns>
+        public async Task ShutdownAsync() {
+            handle.ShutdownAndNotify(serverShutdownHandler);
+            await shutdownTcs.Task;
+            handle.Dispose();
+        }
+
+        public void Kill() {
+            handle.Dispose();
+        }
+
+        private async Task StartHandlingRpcs() {
+            while (true)
+            {
+                await Task.Factory.StartNew(RunRpc);
+            }
+        }
+
+        private void AllowOneRpc()
+        {
+            AssertCallOk(handle.RequestCall(newRpcHandler));
+        }
+
+        private void HandleNewRpc(IntPtr eventPtr)
+        {
+            try
+            {
+                var ev = new EventSafeHandleNotOwned(eventPtr);
+                var rpcInfo = new NewRpcInfo(ev.GetCall(), ev.GetServerRpcNewMethod());
+
+                // after server shutdown, the callback returns with null call
+                if (!rpcInfo.Call.IsInvalid) {
+                    newRpcQueue.Add(rpcInfo);
+                }
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Caught exception in a native handler: " + e);
+            }
+        }
+
+        private void HandleServerShutdown(IntPtr eventPtr)
+        {
+            try
+            {
+                shutdownTcs.SetResult(null);
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Caught exception in a native handler: " + e);
+            }
+        }
+
+        private static void AssertCallOk(GRPCCallError callError)
+        {
+            Trace.Assert(callError == GRPCCallError.GRPC_CALL_OK, "Status not GRPC_CALL_OK");
+        }
+
+        private static CompletionQueueSafeHandle GetCompletionQueue()
+        {
+            return GrpcEnvironment.ThreadPool.CompletionQueue;
+        }
+
+        private struct NewRpcInfo
+        {
+            private CallSafeHandle call;
+            private string method;
+
+            public NewRpcInfo(CallSafeHandle call, string method)
+            {
+                this.call = call;
+                this.method = method;
+            }
+
+            public CallSafeHandle Call
+            {
+                get
+                {
+                    return this.call;
+                }
+            }
+
+            public string Method
+            {
+                get
+                {
+                    return this.method;
+                }
+            }
+        }
+    }
+}