From 376887d213f597da5abf90356ff449cb29e204ee Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 08:27:03 -0700 Subject: [PATCH 01/10] Split event notification out of ev_epoll_linux.c --- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + src/core/lib/iomgr/ev_epoll_linux.c | 219 +---------------- src/core/lib/iomgr/lockfree_event.c | 224 ++++++++++++++++++ src/core/lib/iomgr/lockfree_event.h | 52 ++++ src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util.vcxproj.filters | 6 + .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + 25 files changed, 363 insertions(+), 210 deletions(-) create mode 100644 src/core/lib/iomgr/lockfree_event.c create mode 100644 src/core/lib/iomgr/lockfree_event.h diff --git a/CMakeLists.txt b/CMakeLists.txt index acf47e5bfec..ca2b2598428 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -943,6 +943,7 @@ add_library(grpc src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c @@ -1265,6 +1266,7 @@ add_library(grpc_cronet src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c @@ -1573,6 +1575,7 @@ add_library(grpc_test_util src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c @@ -1833,6 +1836,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c @@ -2249,6 +2253,7 @@ add_library(grpc++ src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c @@ -2579,6 +2584,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c @@ -3277,6 +3283,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c src/core/lib/iomgr/load_file.c + src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c src/core/lib/iomgr/polling_entity.c src/core/lib/iomgr/pollset_set_uv.c diff --git a/Makefile b/Makefile index 5ad05570f93..1eeae6071eb 100644 --- a/Makefile +++ b/Makefile @@ -2845,6 +2845,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ @@ -3165,6 +3166,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ @@ -3472,6 +3474,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ @@ -3704,6 +3707,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ @@ -4097,6 +4101,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ @@ -4435,6 +4440,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ @@ -5125,6 +5131,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ diff --git a/binding.gyp b/binding.gyp index 3444fa53dba..3df6d0aa2f4 100644 --- a/binding.gyp +++ b/binding.gyp @@ -685,6 +685,7 @@ 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', 'src/core/lib/iomgr/load_file.c', + 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', 'src/core/lib/iomgr/polling_entity.c', 'src/core/lib/iomgr/pollset_set_uv.c', diff --git a/build.yaml b/build.yaml index c294fc9185c..c8098c51e5c 100644 --- a/build.yaml +++ b/build.yaml @@ -208,6 +208,7 @@ filegroups: - src/core/lib/iomgr/iomgr_internal.h - src/core/lib/iomgr/iomgr_posix.h - src/core/lib/iomgr/load_file.h + - src/core/lib/iomgr/lockfree_event.h - src/core/lib/iomgr/network_status_tracker.h - src/core/lib/iomgr/polling_entity.h - src/core/lib/iomgr/pollset.h @@ -320,6 +321,7 @@ filegroups: - src/core/lib/iomgr/iomgr_uv.c - src/core/lib/iomgr/iomgr_windows.c - src/core/lib/iomgr/load_file.c + - src/core/lib/iomgr/lockfree_event.c - src/core/lib/iomgr/network_status_tracker.c - src/core/lib/iomgr/polling_entity.c - src/core/lib/iomgr/pollset_set_uv.c diff --git a/config.m4 b/config.m4 index 28467810313..9470e43aaa0 100644 --- a/config.m4 +++ b/config.m4 @@ -119,6 +119,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ + src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_uv.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 83c19afc243..2e260189fbd 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -290,6 +290,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr_internal.h', 'src/core/lib/iomgr/iomgr_posix.h', 'src/core/lib/iomgr/load_file.h', + 'src/core/lib/iomgr/lockfree_event.h', 'src/core/lib/iomgr/network_status_tracker.h', 'src/core/lib/iomgr/polling_entity.h', 'src/core/lib/iomgr/pollset.h', @@ -494,6 +495,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', 'src/core/lib/iomgr/load_file.c', + 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', 'src/core/lib/iomgr/polling_entity.c', 'src/core/lib/iomgr/pollset_set_uv.c', @@ -740,6 +742,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr_internal.h', 'src/core/lib/iomgr/iomgr_posix.h', 'src/core/lib/iomgr/load_file.h', + 'src/core/lib/iomgr/lockfree_event.h', 'src/core/lib/iomgr/network_status_tracker.h', 'src/core/lib/iomgr/polling_entity.h', 'src/core/lib/iomgr/pollset.h', diff --git a/grpc.gemspec b/grpc.gemspec index a3a5870761a..1165a91b6f3 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -206,6 +206,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/iomgr_internal.h ) s.files += %w( src/core/lib/iomgr/iomgr_posix.h ) s.files += %w( src/core/lib/iomgr/load_file.h ) + s.files += %w( src/core/lib/iomgr/lockfree_event.h ) s.files += %w( src/core/lib/iomgr/network_status_tracker.h ) s.files += %w( src/core/lib/iomgr/polling_entity.h ) s.files += %w( src/core/lib/iomgr/pollset.h ) @@ -410,6 +411,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/iomgr_uv.c ) s.files += %w( src/core/lib/iomgr/iomgr_windows.c ) s.files += %w( src/core/lib/iomgr/load_file.c ) + s.files += %w( src/core/lib/iomgr/lockfree_event.c ) s.files += %w( src/core/lib/iomgr/network_status_tracker.c ) s.files += %w( src/core/lib/iomgr/polling_entity.c ) s.files += %w( src/core/lib/iomgr/pollset_set_uv.c ) diff --git a/package.xml b/package.xml index 6957ecca731..c6b86a66a2b 100644 --- a/package.xml +++ b/package.xml @@ -215,6 +215,7 @@ + @@ -419,6 +420,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index e5cf54f10a9..a7105a62e4d 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -56,6 +56,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" @@ -141,52 +142,11 @@ struct grpc_fd { Ref/Unref by two to avoid altering the orphaned bit */ gpr_atm refst; - /* Internally stores data of type (grpc_error *). If the FD is shutdown, this - contains reason for shutdown (i.e a pointer to grpc_error) ORed with - FD_SHUTDOWN_BIT. Since address allocations are word-aligned, the lower bit - of (grpc_error *) addresses is guaranteed to be zero. Even if the - (grpc_error *), is of special types like GRPC_ERROR_NONE, GRPC_ERROR_OOM - etc, the lower bit is guaranteed to be zero. - - Once an fd is shutdown, any pending or future read/write closures on the - fd should fail */ - gpr_atm shutdown_error; - /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer valid */ bool orphaned; - /* Closures to call when the fd is readable or writable respectively. These - fields contain one of the following values: - CLOSURE_READY : The fd has an I/O event of interest but there is no - closure yet to execute - - CLOSURE_NOT_READY : The fd has no I/O event of interest - - closure ptr : The closure to be executed when the fd has an I/O - event of interest - - shutdown_error | FD_SHUTDOWN_BIT : - 'shutdown_error' field ORed with FD_SHUTDOWN_BIT. - This indicates that the fd is shutdown. Since all - memory allocations are word-aligned, the lower two - bits of the shutdown_error pointer are always 0. So - it is safe to OR these with FD_SHUTDOWN_BIT - - Valid state transitions: - - <-----3------ CLOSURE_NOT_READY ----1----> CLOSURE_READY - | | ^ | ^ | | - | | | | | | | - | +--------------4----------+ 6 +---------2---------------+ | - | | | - | v | - +-----5-------> [shutdown_error | FD_SHUTDOWN_BIT] <----7---------+ - - For 1, 4 : See set_ready() function - For 2, 3 : See notify_on() function - For 5,6,7: See set_shutdown() function */ gpr_atm read_closure; gpr_atm write_closure; @@ -218,11 +178,6 @@ static void fd_unref(grpc_fd *fd); static void fd_global_init(void); static void fd_global_shutdown(void); -#define CLOSURE_NOT_READY ((gpr_atm)0) -#define CLOSURE_READY ((gpr_atm)2) - -#define FD_SHUTDOWN_BIT 1 - /******************************************************************************* * Polling island Declarations */ @@ -949,10 +904,8 @@ static void unref_by(grpc_fd *fd, int n) { fd_freelist = fd; grpc_iomgr_unregister_object(&fd->iomgr_object); - grpc_error *err = (grpc_error *)gpr_atm_acq_load(&fd->shutdown_error); - /* Clear the least significant bit if it set (in case fd was shutdown) */ - err = (grpc_error *)((intptr_t)err & ~FD_SHUTDOWN_BIT); - GRPC_ERROR_UNREF(err); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); gpr_mu_unlock(&fd_freelist_mu); } else { @@ -1018,8 +971,8 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->fd = fd; gpr_atm_no_barrier_store(&new_fd->shutdown_error, (gpr_atm)GRPC_ERROR_NONE); new_fd->orphaned = false; - gpr_atm_no_barrier_store(&new_fd->read_closure, CLOSURE_NOT_READY); - gpr_atm_no_barrier_store(&new_fd->write_closure, CLOSURE_NOT_READY); + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = NULL; @@ -1105,153 +1058,6 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_ERROR_UNREF(error); } -static void notify_on(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, - grpc_closure *closure) { - while (true) { - gpr_atm curr = gpr_atm_no_barrier_load(state); - switch (curr) { - case CLOSURE_NOT_READY: { - /* CLOSURE_NOT_READY -> . - - We're guaranteed by API that there's an acquire barrier before here, - so there's no need to double-dip and this can be a release-only. - - The release itself pairs with the acquire half of a set_ready full - barrier. */ - if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { - return; /* Successful. Return */ - } - - break; /* retry */ - } - - case CLOSURE_READY: { - /* Change the state to CLOSURE_NOT_READY. Schedule the closure if - successful. If not, the state most likely transitioned to shutdown. - We should retry. - - This can be a no-barrier cas since the state is being transitioned to - CLOSURE_NOT_READY; set_ready and set_shutdown do not schedule any - closure when transitioning out of CLOSURE_NO_READY state (i.e there - is no other code that needs to 'happen-after' this) */ - if (gpr_atm_no_barrier_cas(state, CLOSURE_READY, CLOSURE_NOT_READY)) { - grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); - return; /* Successful. Return */ - } - - break; /* retry */ - } - - default: { - /* 'curr' is either a closure or the fd is shutdown(in which case 'curr' - contains a pointer to the shutdown-error). If the fd is shutdown, - schedule the closure with the shutdown error */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - grpc_error *shutdown_err = (grpc_error *)(curr & ~FD_SHUTDOWN_BIT); - grpc_closure_sched(exec_ctx, closure, - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "FD Shutdown", &shutdown_err, 1)); - return; - } - - /* There is already a closure!. This indicates a bug in the code */ - gpr_log(GPR_ERROR, - "notify_on called with a previous callback still pending"); - abort(); - } - } - } - - GPR_UNREACHABLE_CODE(return ); -} - -static void set_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state, - grpc_error *shutdown_err) { - gpr_atm new_state = (gpr_atm)shutdown_err | FD_SHUTDOWN_BIT; - - while (true) { - gpr_atm curr = gpr_atm_no_barrier_load(state); - switch (curr) { - case CLOSURE_READY: - case CLOSURE_NOT_READY: - /* Need a full barrier here so that the initial load in notify_on - doesn't need a barrier */ - if (gpr_atm_full_cas(state, curr, new_state)) { - return; /* early out */ - } - break; /* retry */ - - default: { - /* 'curr' is either a closure or the fd is already shutdown */ - - /* If fd is already shutdown, we are done */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - return; - } - - /* Fd is not shutdown. Schedule the closure and move the state to - shutdown state. - Needs an acquire to pair with setting the closure (and get a - happens-after on that edge), and a release to pair with anything - loading the shutdown state. */ - if (gpr_atm_full_cas(state, curr, new_state)) { - grpc_closure_sched(exec_ctx, (grpc_closure *)curr, - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "FD Shutdown", &shutdown_err, 1)); - return; - } - - /* 'curr' was a closure but now changed to a different state. We will - have to retry */ - break; - } - } - } - - GPR_UNREACHABLE_CODE(return ); -} - -static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, gpr_atm *state) { - while (true) { - gpr_atm curr = gpr_atm_no_barrier_load(state); - - switch (curr) { - case CLOSURE_READY: { - /* Already ready. We are done here */ - return; - } - - case CLOSURE_NOT_READY: { - /* No barrier required as we're transitioning to a state that does not - involve a closure */ - if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, CLOSURE_READY)) { - return; /* early out */ - } - break; /* retry */ - } - - default: { - /* 'curr' is either a closure or the fd is shutdown */ - if ((curr & FD_SHUTDOWN_BIT) > 0) { - /* The fd is shutdown. Do nothing */ - return; - } - /* Full cas: acquire pairs with this cas' release in the event of a - spurious set_ready; release pairs with this or the acquire in - notify_on (or set_shutdown) */ - else if (gpr_atm_full_cas(state, curr, CLOSURE_NOT_READY)) { - grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); - return; - } - /* else the state changed again (only possible by either a racing - set_ready or set_shutdown functions. In both these cases, the closure - would have been scheduled for execution. So we are done here */ - return; - } - } - } -} - static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); @@ -1259,23 +1065,16 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, } static bool fd_is_shutdown(grpc_fd *fd) { - grpc_error *err = (grpc_error *)gpr_atm_acq_load(&fd->shutdown_error); - return (((intptr_t)err & FD_SHUTDOWN_BIT) > 0); + return grpc_lfev_is_shutdown(&fd->read_closure); } /* Might be called multiple times */ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { - /* Store the shutdown error ORed with FD_SHUTDOWN_BIT in fd->shutdown_error */ - if (gpr_atm_rel_cas(&fd->shutdown_error, (gpr_atm)GRPC_ERROR_NONE, - (gpr_atm)why | FD_SHUTDOWN_BIT)) { + if (grpc_lfev_set_shutdown(&fd->read_closure, GRPC_ERROR_REF(why))) { shutdown(fd->fd, SHUT_RDWR); - - set_shutdown(exec_ctx, fd, &fd->read_closure, why); - set_shutdown(exec_ctx, fd, &fd->write_closure, why); - } else { - /* Shutdown already called */ - GRPC_ERROR_UNREF(why); + grpc_lfev_set_shutdown(&fd->write_closure, GRPC_ERROR_REF(why)); } + GRPC_ERROR_UNREF(why); } static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c new file mode 100644 index 00000000000..a9742982f68 --- /dev/null +++ b/src/core/lib/iomgr/lockfree_event.c @@ -0,0 +1,224 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/lockfree_event.h" + +#include + +/* 'state' holds the to call when the fd is readable or writable respectively. + It can contain one of the following values: + CLOSURE_READY : The fd has an I/O event of interest but there is no + closure yet to execute + + CLOSURE_NOT_READY : The fd has no I/O event of interest + + closure ptr : The closure to be executed when the fd has an I/O + event of interest + + shutdown_error | FD_SHUTDOWN_BIT : + 'shutdown_error' field ORed with FD_SHUTDOWN_BIT. + This indicates that the fd is shutdown. Since all + memory allocations are word-aligned, the lower two + bits of the shutdown_error pointer are always 0. So + it is safe to OR these with FD_SHUTDOWN_BIT + + Valid state transitions: + + <-----3------ CLOSURE_NOT_READY ----1----> CLOSURE_READY + | | ^ | ^ | | + | | | | | | | + | +--------------4----------+ 6 +---------2---------------+ | + | | | + | v | + +-----5-------> [shutdown_error | FD_SHUTDOWN_BIT] <----7---------+ + + For 1, 4 : See grpc_lfev_set_ready() function + For 2, 3 : See grpc_lfev_notify_on() function + For 5,6,7: See grpc_lfev_set_shutdown() function */ + +#define CLOSURE_NOT_READY ((gpr_atm)0) +#define CLOSURE_READY ((gpr_atm)2) + +#define FD_SHUTDOWN_BIT 1 + +void grpc_lfev_init(gpr_atm *state) { + gpr_atm_no_barrier_store(state, CLOSURE_NOT_READY); +} + +void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_closure *closure) { + while (true) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + switch (curr) { + case CLOSURE_NOT_READY: { + /* CLOSURE_NOT_READY -> . + + We're guaranteed by API that there's an acquire barrier before here, + so there's no need to double-dip and this can be a release-only. + + The release itself pairs with the acquire half of a set_ready full + barrier. */ + if (gpr_atm_rel_cas(state, CLOSURE_NOT_READY, (gpr_atm)closure)) { + return; /* Successful. Return */ + } + + break; /* retry */ + } + + case CLOSURE_READY: { + /* Change the state to CLOSURE_NOT_READY. Schedule the closure if + successful. If not, the state most likely transitioned to shutdown. + We should retry. + + This can be a no-barrier cas since the state is being transitioned to + CLOSURE_NOT_READY; set_ready and set_shutdown do not schedule any + closure when transitioning out of CLOSURE_NO_READY state (i.e there + is no other code that needs to 'happen-after' this) */ + if (gpr_atm_no_barrier_cas(state, CLOSURE_READY, CLOSURE_NOT_READY)) { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); + return; /* Successful. Return */ + } + + break; /* retry */ + } + + default: { + /* 'curr' is either a closure or the fd is shutdown(in which case 'curr' + contains a pointer to the shutdown-error). If the fd is shutdown, + schedule the closure with the shutdown error */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + grpc_error *shutdown_err = (grpc_error *)(curr & ~FD_SHUTDOWN_BIT); + grpc_closure_sched(exec_ctx, closure, + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "FD Shutdown", &shutdown_err, 1)); + return; + } + + /* There is already a closure!. This indicates a bug in the code */ + gpr_log(GPR_ERROR, + "notify_on called with a previous callback still pending"); + abort(); + } + } + } + + GPR_UNREACHABLE_CODE(return ); +} + +bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_error *shutdown_err) { + gpr_atm new_state = (gpr_atm)shutdown_err | FD_SHUTDOWN_BIT; + + while (true) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + switch (curr) { + case CLOSURE_READY: + case CLOSURE_NOT_READY: + /* Need a full barrier here so that the initial load in notify_on + doesn't need a barrier */ + if (gpr_atm_full_cas(state, curr, new_state)) { + return true; /* early out */ + } + break; /* retry */ + + default: { + /* 'curr' is either a closure or the fd is already shutdown */ + + /* If fd is already shutdown, we are done */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + GRPC_ERROR_UNREF(shutdown_err); + return false; + } + + /* Fd is not shutdown. Schedule the closure and move the state to + shutdown state. + Needs an acquire to pair with setting the closure (and get a + happens-after on that edge), and a release to pair with anything + loading the shutdown state. */ + if (gpr_atm_full_cas(state, curr, new_state)) { + grpc_closure_sched(exec_ctx, (grpc_closure *)curr, + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "FD Shutdown", &shutdown_err, 1)); + return true; + } + + /* 'curr' was a closure but now changed to a different state. We will + have to retry */ + break; + } + } + } + + GPR_UNREACHABLE_CODE(return false); +} + +void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) { + while (true) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + + switch (curr) { + case CLOSURE_READY: { + /* Already ready. We are done here */ + return; + } + + case CLOSURE_NOT_READY: { + /* No barrier required as we're transitioning to a state that does not + involve a closure */ + if (gpr_atm_no_barrier_cas(state, CLOSURE_NOT_READY, CLOSURE_READY)) { + return; /* early out */ + } + break; /* retry */ + } + + default: { + /* 'curr' is either a closure or the fd is shutdown */ + if ((curr & FD_SHUTDOWN_BIT) > 0) { + /* The fd is shutdown. Do nothing */ + return; + } + /* Full cas: acquire pairs with this cas' release in the event of a + spurious set_ready; release pairs with this or the acquire in + notify_on (or set_shutdown) */ + else if (gpr_atm_full_cas(state, curr, CLOSURE_NOT_READY)) { + grpc_closure_sched(exec_ctx, (grpc_closure *)curr, GRPC_ERROR_NONE); + return; + } + /* else the state changed again (only possible by either a racing + set_ready or set_shutdown functions. In both these cases, the closure + would have been scheduled for execution. So we are done here */ + return; + } + } + } +} diff --git a/src/core/lib/iomgr/lockfree_event.h b/src/core/lib/iomgr/lockfree_event.h new file mode 100644 index 00000000000..32eb7554f96 --- /dev/null +++ b/src/core/lib/iomgr/lockfree_event.h @@ -0,0 +1,52 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H +#define GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H + +/* Lock free event notification for file descriptors */ + +#include "src/core/lib/iomgr/exec_ctx.h" + +void grpc_lfev_init(gpr_atm *state); +void grpc_lfev_destroy(gpr_atm *state); +bool grpc_lfev_is_shutdown(gpr_atm *state); + +void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_closure *closure); +/* Returns true on first successful shutdown */ +bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state, + grpc_error *shutdown_err); +void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state); + +#endif /* GRPC_CORE_LIB_IOMGR_LOCKFREE_EVENT_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 3bcbe667e2c..0d0a5fb0887 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -113,6 +113,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', 'src/core/lib/iomgr/load_file.c', + 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', 'src/core/lib/iomgr/polling_entity.c', 'src/core/lib/iomgr/pollset_set_uv.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 209d5445dbc..afab2296a02 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -971,6 +971,8 @@ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/load_file.h \ +src/core/lib/iomgr/lockfree_event.c \ +src/core/lib/iomgr/lockfree_event.h \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/network_status_tracker.h \ src/core/lib/iomgr/polling_entity.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 6903a0e5d5a..ee1fd1b242f 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1092,6 +1092,8 @@ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/load_file.h \ +src/core/lib/iomgr/lockfree_event.c \ +src/core/lib/iomgr/lockfree_event.h \ src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/network_status_tracker.h \ src/core/lib/iomgr/polling_entity.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index ae5120d87c1..fa62d0aaaca 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7549,6 +7549,7 @@ "src/core/lib/iomgr/iomgr_internal.h", "src/core/lib/iomgr/iomgr_posix.h", "src/core/lib/iomgr/load_file.h", + "src/core/lib/iomgr/lockfree_event.h", "src/core/lib/iomgr/network_status_tracker.h", "src/core/lib/iomgr/polling_entity.h", "src/core/lib/iomgr/pollset.h", @@ -7710,6 +7711,8 @@ "src/core/lib/iomgr/iomgr_windows.c", "src/core/lib/iomgr/load_file.c", "src/core/lib/iomgr/load_file.h", + "src/core/lib/iomgr/lockfree_event.c", + "src/core/lib/iomgr/lockfree_event.h", "src/core/lib/iomgr/network_status_tracker.c", "src/core/lib/iomgr/network_status_tracker.h", "src/core/lib/iomgr/polling_entity.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index caa22a019db..cae44c5e234 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -412,6 +412,7 @@ + @@ -647,6 +648,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 6fc1c969309..36da089470b 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -232,6 +232,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -968,6 +971,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 674818182e8..4d284006e67 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -406,6 +406,7 @@ + @@ -631,6 +632,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 2b9a5b13c1a..9d641553a2b 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -217,6 +217,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -935,6 +938,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 948af2f29ee..dd6fdd861e4 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -335,6 +335,7 @@ + @@ -578,6 +579,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index d43b0e3f83e..51c9b7201dc 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -112,6 +112,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -902,6 +905,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index c675cda122f..b1db7e5d15d 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -230,6 +230,7 @@ + @@ -419,6 +420,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index c6bd2d6c9f1..a7ecbea87c7 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -169,6 +169,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -680,6 +683,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 8799359bb76..8614b115898 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -325,6 +325,7 @@ + @@ -546,6 +547,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index cf2def4ff65..e7aa9fb8d6f 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -115,6 +115,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -815,6 +818,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From 70652147ceb2bedff5ac3b491f0f885f5a997cc4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 08:31:23 -0700 Subject: [PATCH 02/10] Update function names --- src/core/lib/iomgr/ev_epoll_linux.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index a7105a62e4d..6bdd599c23f 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1079,12 +1079,12 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { - notify_on(exec_ctx, fd, &fd->read_closure, closure); + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); } static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { - notify_on(exec_ctx, fd, &fd->write_closure, closure); + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { @@ -1274,7 +1274,7 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *notifier) { - set_ready(exec_ctx, fd, &fd->read_closure); + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); /* Note, it is possible that fd_become_readable might be called twice with different 'notifier's when an fd becomes readable and it is in two epoll @@ -1286,7 +1286,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - set_ready(exec_ctx, fd, &fd->write_closure); + grpc_lfev_set_ready(exec_ctx, fd, &fd->write_closure); } static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, From e16372ba4b442bcf0c1330c01f0e7f772a3b05cd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 08:51:39 -0700 Subject: [PATCH 03/10] Cleanup to make compile --- src/core/lib/iomgr/ev_epoll_linux.c | 8 ++++---- src/core/lib/iomgr/lockfree_event.c | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 6bdd599c23f..6db8e1a77cb 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -969,7 +969,6 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; - gpr_atm_no_barrier_store(&new_fd->shutdown_error, (gpr_atm)GRPC_ERROR_NONE); new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); @@ -1070,9 +1069,10 @@ static bool fd_is_shutdown(grpc_fd *fd) { /* Might be called multiple times */ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { - if (grpc_lfev_set_shutdown(&fd->read_closure, GRPC_ERROR_REF(why))) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { shutdown(fd->fd, SHUT_RDWR); - grpc_lfev_set_shutdown(&fd->write_closure, GRPC_ERROR_REF(why)); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); } GRPC_ERROR_UNREF(why); } @@ -1286,7 +1286,7 @@ static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - grpc_lfev_set_ready(exec_ctx, fd, &fd->write_closure); + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c index a9742982f68..17e3bbf7278 100644 --- a/src/core/lib/iomgr/lockfree_event.c +++ b/src/core/lib/iomgr/lockfree_event.c @@ -69,12 +69,26 @@ #define CLOSURE_NOT_READY ((gpr_atm)0) #define CLOSURE_READY ((gpr_atm)2) -#define FD_SHUTDOWN_BIT 1 +#define FD_SHUTDOWN_BIT ((gpr_atm)1) void grpc_lfev_init(gpr_atm *state) { gpr_atm_no_barrier_store(state, CLOSURE_NOT_READY); } +void grpc_lfev_destroy(gpr_atm *state) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + if (curr & FD_SHUTDOWN_BIT) { + GRPC_ERROR_UNREF((grpc_error *)(curr & ~FD_SHUTDOWN_BIT)); + } else { + GPR_ASSERT(curr == CLOSURE_NOT_READY || curr == CLOSURE_READY); + } +} + +bool grpc_lfev_is_shutdown(gpr_atm *state) { + gpr_atm curr = gpr_atm_no_barrier_load(state); + return (curr & FD_SHUTDOWN_BIT) != 0; +} + void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, grpc_closure *closure) { while (true) { From 3de12630da62b83defb1987f6627733174169222 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 10:02:31 -0700 Subject: [PATCH 04/10] Ensure cleanup before orphaning --- test/core/iomgr/ev_epoll_linux_test.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 5f8124aedae..0856023b14f 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -346,11 +346,13 @@ static void test_threading_wakeup(grpc_exec_ctx *exec_ctx, void *arg, threading_shared *shared = arg; ++shared->wakeups; ++thread_wakeups; - GPR_ASSERT(GRPC_LOG_IF_ERROR( - "consume_wakeup", grpc_wakeup_fd_consume_wakeup(shared->wakeup_fd))); - grpc_fd_notify_on_read(exec_ctx, shared->wakeup_desc, &shared->on_wakeup); - GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_next", - grpc_wakeup_fd_wakeup(shared->wakeup_fd))); + if (error == GRPC_ERROR_NONE) { + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "consume_wakeup", grpc_wakeup_fd_consume_wakeup(shared->wakeup_fd))); + grpc_fd_notify_on_read(exec_ctx, shared->wakeup_desc, &shared->on_wakeup); + GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_next", + grpc_wakeup_fd_wakeup(shared->wakeup_fd))); + } } static void test_threading(void) { @@ -387,6 +389,7 @@ static void test_threading(void) { grpc_wakeup_fd_destroy(&fd); { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_fd_shutdown(&exec_ctx, shared.wakeup_desc, GRPC_ERROR_CANCELLED); grpc_fd_orphan(&exec_ctx, shared.wakeup_desc, NULL, NULL, "done"); grpc_pollset_shutdown(&exec_ctx, shared.pollset, grpc_closure_create(destroy_pollset, shared.pollset, From 6badbb74b69d365ff775753ca0be9d75973aebd3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 10:03:53 -0700 Subject: [PATCH 05/10] Update BUILD --- BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BUILD b/BUILD index c4632fc0a24..c9bdae06c40 100644 --- a/BUILD +++ b/BUILD @@ -457,6 +457,7 @@ grpc_cc_library( "src/core/lib/iomgr/endpoint_pair_windows.c", "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll_linux.c", + "src/core/lib/iomgr/lockfree_event.c", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_posix.c", "src/core/lib/iomgr/exec_ctx.c", @@ -583,6 +584,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/lockfree_event.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", From 3569a76dc5f25ddf84b791a1073021cd90aa674d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 11:47:10 -0700 Subject: [PATCH 06/10] Explicitly include atm to fix Windows build --- src/core/lib/iomgr/lockfree_event.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/lockfree_event.h b/src/core/lib/iomgr/lockfree_event.h index 32eb7554f96..1d9119204ca 100644 --- a/src/core/lib/iomgr/lockfree_event.h +++ b/src/core/lib/iomgr/lockfree_event.h @@ -36,6 +36,8 @@ /* Lock free event notification for file descriptors */ +#include + #include "src/core/lib/iomgr/exec_ctx.h" void grpc_lfev_init(gpr_atm *state); From 2c3d835d66d5e26aa2695a148cae9c255e8e2a16 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:09:52 -0700 Subject: [PATCH 07/10] Implement gpr_atm_full_cas for Windows --- include/grpc/impl/codegen/atm_windows.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/grpc/impl/codegen/atm_windows.h b/include/grpc/impl/codegen/atm_windows.h index b8f63da7587..a533651f6f9 100644 --- a/include/grpc/impl/codegen/atm_windows.h +++ b/include/grpc/impl/codegen/atm_windows.h @@ -95,6 +95,16 @@ static __inline int gpr_atm_rel_cas(gpr_atm *p, gpr_atm o, gpr_atm n) { #endif } +static __inline int gpr_atm_full_cas(gpr_atm *p, gpr_atm o, gpr_atm n) { +#ifdef GPR_ARCH_64 + return o == (gpr_atm)InterlockedCompareExchange64((volatile LONGLONG *)p, + (LONGLONG)n, (LONGLONG)o); +#else + return o == (gpr_atm)InterlockedCompareExchange((volatile LONG *)p, (LONG)n, + (LONG)o); +#endif +} + static __inline gpr_atm gpr_atm_no_barrier_fetch_add(gpr_atm *p, gpr_atm delta) { /* Use the CAS operation to get pointer-sized fetch and add */ From 025f110ed26afe600c10a4af9648bd6933174168 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 10 Apr 2017 13:07:37 -0700 Subject: [PATCH 08/10] Node: fix Windows warning in binding.gyp for Win10 console --- binding.gyp | 5 +++-- templates/binding.gyp.template | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/binding.gyp b/binding.gyp index 3444fa53dba..17a86335581 100644 --- a/binding.gyp +++ b/binding.gyp @@ -519,9 +519,10 @@ # the OpenSSL headers, from the downloaded Node development package, # which is typically located in `.node-gyp` in your home directory. 'target_name': 'WINDOWS_BUILD_WARNING', - 'actions': [ + 'rules': [ { - 'action_name': 'WINDOWS_BUILD_WARNING', + 'rule_name': 'WINDOWS_BUILD_WARNING', + 'extension': 'S', 'inputs': [ 'package.json' ], diff --git a/templates/binding.gyp.template b/templates/binding.gyp.template index aeeb56b9a6a..55a91c5b93d 100644 --- a/templates/binding.gyp.template +++ b/templates/binding.gyp.template @@ -217,9 +217,10 @@ # the OpenSSL headers, from the downloaded Node development package, # which is typically located in `.node-gyp` in your home directory. 'target_name': 'WINDOWS_BUILD_WARNING', - 'actions': [ + 'rules': [ { - 'action_name': 'WINDOWS_BUILD_WARNING', + 'rule_name': 'WINDOWS_BUILD_WARNING', + 'extension': 'S', 'inputs': [ 'package.json' ], From 64657762f09939f77a6ed6cd635947ab94c6dae0 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 10 Apr 2017 22:54:14 -0700 Subject: [PATCH 09/10] Generate projects --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fcd2d01310..7ef7311627e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8981,7 +8981,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_cq_multiple_threads test/cpp/microbenchmarks/bm_cq_multiple_threads.cc - third_party/googletest/src/gtest-all.cc + third_party/googletest/googletest/src/gtest-all.cc ) @@ -8998,8 +8998,8 @@ target_include_directories(bm_cq_multiple_threads PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest PRIVATE ${_gRPC_PROTO_GENS_DIR} ) From 108a109c02ae38d2253673edde592b17c54fb49c Mon Sep 17 00:00:00 2001 From: Chris Trimble Date: Wed, 11 Jan 2017 18:05:08 -0800 Subject: [PATCH 10/10] Google Compute Metadata hostname fix From within containers (or possibly other scenarios), the metadata hostname might not resolve on GCP. Setting this to the full path guarantees it will resolve. --- src/core/lib/security/credentials/credentials.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/security/credentials/credentials.h b/src/core/lib/security/credentials/credentials.h index 510b79552a3..89b8e3c0b38 100644 --- a/src/core/lib/security/credentials/credentials.h +++ b/src/core/lib/security/credentials/credentials.h @@ -71,7 +71,7 @@ typedef enum { #define GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS 60 -#define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata" +#define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata.google.internal" #define GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH \ "/computeMetadata/v1/instance/service-accounts/default/token"