From dd86b69b8b8d7092198249acdef50ede1e7bfac5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 10:43:11 -0700 Subject: [PATCH 001/177] Stubbed out new polling engine --- src/core/lib/iomgr/ev_epollex_linux.c | 635 ++++++++++++++++++++++++++ src/core/lib/iomgr/ev_epollex_linux.h | 42 ++ src/core/lib/iomgr/ev_posix.c | 2 + 3 files changed, 679 insertions(+) create mode 100644 src/core/lib/iomgr/ev_epollex_linux.c create mode 100644 src/core/lib/iomgr/ev_epollex_linux.h diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c new file mode 100644 index 00000000000..71e8b7e4fb7 --- /dev/null +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -0,0 +1,635 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* Uncomment the following to enable extra checks on poll_object operations */ +/* #define PO_DEBUG */ + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +/******************************************************************************* + * Fd Declarations + */ + +#define FD_FROM_PO(po) ((grpc_fd *)(po)) + +struct grpc_fd { + gpr_mu mu; + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +/******************************************************************************* + * Pollset Declarations + */ +struct grpc_pollset_worker { + /* Thread id of this worker */ + pthread_t pt_id; + + /* Used to prevent a worker from getting kicked multiple times */ + gpr_atm is_kicked; + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset {}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set {}; + +/******************************************************************************* + * Common helpers + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + abort(); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + abort(); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + abort(); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + abort(); + } +} +#endif + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + // grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + abort(); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + abort(); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->mu); + } + + /* Note: It is not really needed to get the new_fd->mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->mu); + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + + gpr_mu_lock(&fd->mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +static void poller_kick_init() {} + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + poller_kick_init(); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + abort(); +} + +static grpc_error *kick_poller(void) { abort(); } + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + abort(); +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) {} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) {} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + abort(); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + abort(); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + abort(); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + abort(); +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + abort(); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + abort(); +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + abort(); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + abort(); +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epollex_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epollex_available()) { + return NULL; + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ + +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.h b/src/core/lib/iomgr/ev_epollex_linux.h new file mode 100644 index 00000000000..4d7cd713df0 --- /dev/null +++ b/src/core/lib/iomgr/ev_epollex_linux.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epollex_linux(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index b5be5504b9d..2663152aeca 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,6 +45,7 @@ #include #include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -65,6 +66,7 @@ typedef struct { } event_engine_factory; static const event_engine_factory g_factories[] = { + {"epollex", grpc_init_epollex_linux}, {"epoll", grpc_init_epoll_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, From e24b24d3c646c40248ea4581f3c5b03597797544 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:05:45 -0700 Subject: [PATCH 002/177] Implement pollset for epollex --- src/core/lib/iomgr/ev_epoll_linux.c | 2 + src/core/lib/iomgr/ev_epollex_linux.c | 279 +++++++++++++++++++++++--- src/core/lib/iomgr/ev_poll_posix.c | 2 + src/core/lib/iomgr/pollset.h | 5 +- src/core/lib/iomgr/pollset_windows.c | 2 + 5 files changed, 254 insertions(+), 36 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 6db8e1a77cb..d41c164d71f 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -63,6 +63,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ static int grpc_polling_trace = 0; /* Disabled by default */ #define GRPC_POLLING_TRACE(fmt, ...) \ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 71e8b7e4fb7..0985755a43b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -62,8 +62,9 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -/* Uncomment the following to enable extra checks on poll_object operations */ -/* #define PO_DEBUG */ +#ifndef EPOLLEXCLUSIVE +#define EPOLLEXCLUSIVE (1u << 28) +#endif /* TODO: sreek: Right now, this wakes up all pollers. In future we should make * sure to wake up one polling thread (which can wake up other threads if @@ -85,6 +86,8 @@ struct grpc_fd { Ref/Unref by two to avoid altering the orphaned bit */ gpr_atm refst; + grpc_wakeup_fd workqueue_wakeup_fd; + /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer valid */ @@ -131,16 +134,22 @@ static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { * Pollset Declarations */ struct grpc_pollset_worker { - /* Thread id of this worker */ - pthread_t pt_id; - - /* Used to prevent a worker from getting kicked multiple times */ - gpr_atm is_kicked; - struct grpc_pollset_worker *next; - struct grpc_pollset_worker *prev; + bool kicked; + bool initialized_cv; + gpr_cv cv; + grpc_pollset_worker *next; + grpc_pollset_worker *prev; }; -struct grpc_pollset {}; +struct grpc_pollset { + gpr_mu mu; + int epfd; + int num_pollers; + gpr_atm shutdown_atm; + grpc_closure *shutdown_closure; + grpc_wakeup_fd pollset_wakeup; + grpc_pollset_worker *root_worker; +}; /******************************************************************************* * Pollset-set Declarations @@ -151,6 +160,16 @@ struct grpc_pollset_set {}; * Common helpers */ +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, @@ -400,13 +419,10 @@ static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); -static void poller_kick_init() {} - /* Global state management */ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - poller_kick_init(); return grpc_wakeup_fd_init(&global_wakeup_fd); } @@ -419,12 +435,41 @@ static void pollset_global_shutdown(void) { /* p->mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { - abort(); + if (specific_worker == NULL) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + } else { + return GRPC_ERROR_NONE; + } + } else if (gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + return GRPC_ERROR_NONE; + } else if (specific_worker == p->root_worker) { + return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + } else { + gpr_cv_signal(&specific_worker->cv); + return GRPC_ERROR_NONE; + } } -static grpc_error *kick_poller(void) { abort(); } +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} -static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {} +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + pollset->epfd = epoll_create1(EPOLL_CLOEXEC); + if (pollset->epfd < 0) { + GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); + } + pollset->num_pollers = 0; + gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); + pollset->shutdown_closure = NULL; + GRPC_LOG_IF_ERROR("pollset_init", + grpc_wakeup_fd_init(&pollset->pollset_wakeup)); + pollset->root_worker = NULL; + *mu = &pollset->mu; +} /* Convert a timespec to milliseconds: - Very small or negative poll times are clamped to zero to do a non-blocking @@ -469,33 +514,186 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset) { - abort(); -} - /* pollset->po.mu lock must be held by the caller before calling this */ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_closure *closure) {} + grpc_closure *closure) { + GPR_ASSERT(pollset->shutdown_closure == NULL); + pollset->shutdown_closure = closure; + if (pollset->num_pollers > 0) { + struct epoll_event ev = {.events = EPOLLIN, + .data.ptr = &pollset->pollset_wakeup}; + epoll_ctl(pollset->epfd, EPOLL_CTL_MOD, pollset->pollset_wakeup.read_fd, + &ev); + GRPC_LOG_IF_ERROR("pollset_shutdown", + grpc_wakeup_fd_wakeup(&pollset->pollset_wakeup)); + } else { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); + } +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. */ +static void pollset_destroy(grpc_pollset *pollset) { + gpr_mu_destroy(&pollset->mu); + if (pollset->epfd >= 0) close(pollset->epfd); + grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); +} -/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other - * than destroying the mutexes, there is nothing special that needs to be done - * here */ -static void pollset_destroy(grpc_pollset *pollset) {} +#define MAX_EPOLL_EVENTS 100 -/* pollset->po.mu lock must be held by the caller before calling this. - The function pollset_work() may temporarily release the lock (pollset->po.mu) +static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { + struct epoll_event events[MAX_EPOLL_EVENTS]; + static const char *err_desc = "pollset_poll"; + + if (pollset->epfd < 0) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "epoll fd failed to initialize"); + } + + int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, + poll_deadline_to_millis_timeout(deadline, now)); + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + + grpc_error *error = GRPC_ERROR_NONE; + for (int i = 0; i < r; i++) { + void *data_ptr = events[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pollset->pollset_wakeup) { + /* once we start shutting down we stop consuming the wakeup: + the fd is level triggered and non-exclusive, which should result in all + pollers waking */ + if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } + } else { + grpc_fd *fd = (grpc_fd *)(((intptr_t)data_ptr) & ~(intptr_t)1); + bool is_workqueue = (((intptr_t)data_ptr) & 1) != 0; + bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; + bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; + bool write_ev = (events[i].events & EPOLLOUT) != 0; + if (is_workqueue) { + append_error(&error, + grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), + err_desc); + fd_invoke_workqueue(exec_ctx, fd); + } else { + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + } + + return error; +} + +/* Return true if this thread should poll */ +static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl, + gpr_timespec deadline) { + if (worker_hdl != NULL) { + *worker_hdl = worker; + worker->kicked = false; + if (pollset->root_worker == NULL) { + pollset->root_worker = worker; + worker->next = worker->prev = worker; + worker->initialized_cv = false; + } else { + worker->next = pollset->root_worker; + worker->prev = worker->next->prev; + worker->next->prev = worker->prev->next = worker; + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (pollset->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline)) return false; + if (worker->kicked) return false; + } + } + } + return pollset->shutdown_closure == NULL; +} + +static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl) { + if (worker_hdl != NULL) { + if (worker == pollset->root_worker) { + if (worker == worker->next) { + pollset->root_worker = NULL; + } else { + pollset->root_worker = worker->next; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; + } + } else { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + } +} + +/* pollset->mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->mu) during the course of its execution but it will always re-acquire the lock and ensure that it is held by the time the function returns */ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { - abort(); + grpc_pollset_worker worker; + grpc_error *error = GRPC_ERROR_NONE; + if (begin_worker(pollset, &worker, worker_hdl, deadline)) { + GPR_ASSERT(!pollset->shutdown_closure); + pollset->num_pollers++; + gpr_mu_unlock(&pollset->mu); + error = pollset_poll(exec_ctx, pollset, now, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + pollset->num_pollers--; + if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + } + } + end_worker(pollset, &worker, worker_hdl); + return error; } static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - abort(); + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_add_fd"; + struct epoll_event ev_fd = { + .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { + switch (errno) { + case EEXIST: /* if this fd is already in the epoll set, the workqueue fd + must also be - just return */ + return; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, + .data.ptr = fd}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, + &ev_wq) != 0) { + switch (errno) { + case EEXIST: /* if the workqueue fd is already in the epoll set we're ok - + no need to do anything special */ + break; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + GRPC_LOG_IF_ERROR("pollset_add_fd", error); } /******************************************************************************* @@ -593,15 +791,32 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epollex_available() { +static bool is_epollex_available(void) { int fd = epoll_create1(EPOLL_CLOEXEC); if (fd < 0) { gpr_log( GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epoll polling engine", + "epoll_create1 failed with error: %d. Not using epollex polling engine", fd); return false; } + grpc_wakeup_fd wakeup; + if (!GRPC_LOG_IF_ERROR("check_wakeupfd_for_epollex", + grpc_wakeup_fd_init(&wakeup))) { + return false; + } + struct epoll_event ev = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, + .data.ptr = NULL}; + if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE failed with error: %d. Not using " + "epollex polling engine", + fd); + close(fd); + grpc_wakeup_fd_destroy(&wakeup); + return false; + } + grpc_wakeup_fd_destroy(&wakeup); close(fd); return true; } diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 9834cdd1979..b13ec2cbc1e 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -58,6 +58,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + /******************************************************************************* * FD declarations */ diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index 9bf3cdac89e..6f3a51e7175 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -40,8 +40,6 @@ #include "src/core/lib/iomgr/exec_ctx.h" -#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) - /* A grpc_pollset is a set of file descriptors that a higher level item is interested in. For example: - a server will typically keep a pollset containing all connected channels, @@ -88,8 +86,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec deadline) GRPC_MUST_USE_RESULT; /* Break one polling thread out of polling work for this pollset. - If specific_worker is GRPC_POLLSET_KICK_BROADCAST, kick ALL the workers. - Otherwise, if specific_worker is non-NULL, then kick that worker. */ + If specific_worker is non-NULL, then kick that worker. */ grpc_error *grpc_pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 04c6b717475..6dca37b481c 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -43,6 +43,8 @@ #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_windows.h" +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) + gpr_mu grpc_polling_mu; static grpc_pollset_worker *g_active_poller; static grpc_pollset_worker g_global_root_worker; From a74808604a03682108fc4a6d72384f1f5216849c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:13:41 -0700 Subject: [PATCH 003/177] Fix pollset_init to also add the wakeup fd --- src/core/lib/iomgr/ev_epollex_linux.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0985755a43b..839807c246c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -465,8 +465,16 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->num_pollers = 0; gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); pollset->shutdown_closure = NULL; - GRPC_LOG_IF_ERROR("pollset_init", - grpc_wakeup_fd_init(&pollset->pollset_wakeup)); + if (GRPC_LOG_IF_ERROR("pollset_init", + grpc_wakeup_fd_init(&pollset->pollset_wakeup)) && + pollset->epfd >= 0) { + struct epoll_event ev = {.events = EPOLLIN | EPOLLET, + .data.ptr = &pollset->pollset_wakeup}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, pollset->pollset_wakeup.read_fd, + &ev) != 0) { + GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); + } + } pollset->root_worker = NULL; *mu = &pollset->mu; } From 12f0a582fb28f5a7838aff00328d6729ea458670 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:14:54 -0700 Subject: [PATCH 004/177] Add global wakeup to pollset --- src/core/lib/iomgr/ev_epollex_linux.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 839807c246c..8d3e3bd562e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -461,6 +461,13 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); + } else { + struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, + &ev) != 0) { + GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); + } } pollset->num_pollers = 0; gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); From 9fae6f9cc2939522912eeed3057af461b487ca90 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 16:25:28 -0700 Subject: [PATCH 005/177] Starting to flesh out fd --- src/core/lib/iomgr/ev_epollex_linux.c | 107 ++++++++++++-------------- 1 file changed, 49 insertions(+), 58 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 8d3e3bd562e..b3390c602ec 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -87,6 +87,7 @@ struct grpc_fd { gpr_atm refst; grpc_wakeup_fd workqueue_wakeup_fd; + grpc_closure_scheduler workqueue_scheduler; /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer @@ -106,21 +107,6 @@ struct grpc_fd { grpc_iomgr_object iomgr_object; }; -/* Reference counting for fds */ -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd *fd); -static void fd_unref(grpc_fd *fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - static void fd_global_init(void); static void fd_global_shutdown(void); @@ -170,49 +156,6 @@ static bool append_error(grpc_error **composite, grpc_error *error, return false; } -#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, - const char *file, int line, - const char *reason) { - if (workqueue != NULL) { - abort(); - } - return workqueue; -} - -static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, - const char *file, int line, const char *reason) { - if (workqueue != NULL) { - abort(); - } -} -#else -static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { - if (workqueue != NULL) { - abort(); - } - return workqueue; -} - -static void workqueue_unref(grpc_exec_ctx *exec_ctx, - grpc_workqueue *workqueue) { - if (workqueue != NULL) { - abort(); - } -} -#endif - -static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, - grpc_error *error) { - GPR_TIMER_BEGIN("workqueue.enqueue", 0); - // grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; - abort(); -} - -static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - abort(); -} - /******************************************************************************* * Fd Definitions */ @@ -323,6 +266,10 @@ static grpc_fd *fd_create(int fd, const char *name) { grpc_lfev_init(&new_fd->write_closure); gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + GRPC_LOG_IF_ERROR("fd_create", + grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); + new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -413,6 +360,50 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + ref_by((grpc_fd *)workqueue, 2, file, line, reason); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + unref_by((grpc_fd *)workqueue, 2, file, line, reason); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + ref_by((grpc_fd *)workqueue, 2); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + unref_by((grpc_fd *)workqueue, 2); + } +} +#endif + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - + offsetof(grpc_fd, workqueue_scheduler)); + abort(); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + return &((grpc_fd *)workqueue)->workqueue_scheduler; +} + /******************************************************************************* * Pollset Definitions */ From 0b4c9011846c9f9640eabefcbecc9d8736b3ccc0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 17:19:37 -0700 Subject: [PATCH 006/177] fd-workqueue-melding --- src/core/lib/iomgr/ev_epollex_linux.c | 50 ++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b3390c602ec..608cda4a3bc 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -61,6 +61,7 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/spinlock.h" #ifndef EPOLLEXCLUSIVE #define EPOLLEXCLUSIVE (1u << 28) @@ -86,8 +87,16 @@ struct grpc_fd { Ref/Unref by two to avoid altering the orphaned bit */ gpr_atm refst; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; grpc_closure_scheduler workqueue_scheduler; + /* Spinlock guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_spinlock workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer @@ -269,6 +278,9 @@ static grpc_fd *fd_create(int fd, const char *name) { GRPC_LOG_IF_ERROR("fd_create", grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + new_fd->workqueue_read_mu = GPR_SPINLOCK_INIT; + gpr_mpscq_init(&new_fd->workqueue_items); + gpr_atm_no_barrier_store(&new_fd->workqueue_item_count); new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -392,12 +404,46 @@ static void workqueue_unref(grpc_exec_ctx *exec_ctx, } #endif +static void workqueue_wakeup(grpc_fd *fd) { + GRPC_LOG_IF_ERROR("workqueue_enqueue", + grpc_wakeup_fd_wakeup(&fd->workqueue_wakeup_fd)); +} + static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error) { GPR_TIMER_BEGIN("workqueue.enqueue", 0); grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - offsetof(grpc_fd, workqueue_scheduler)); - abort(); + REF_BY(fd, 2); + gpr_atm last = gpr_atm_no_barrier_fetch_add(&fd->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&fd->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_wakeup(fd); + } + UNREF_BY(fd, 2); +} + +static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + /* handle spurious wakeups */ + if (!gpr_spinlock_trylock(&fd->workqueue_read_mu)) return; + gpr_mpscq_node *n = gpr_mpscq_pop(&fd->workqueue_items); + gpr_spinlock_unlock(&fd->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_wakeup(fd); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_wakeup(fd); + } } static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { @@ -556,8 +602,10 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, "epoll fd failed to initialize"); } + GRPC_SCHEDULING_START_BLOCKING_REGION; int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, poll_deadline_to_millis_timeout(deadline, now)); + GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); grpc_error *error = GRPC_ERROR_NONE; From 40664c7189f2599168edb28aa49f6cd7a6e9bb40 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 08:19:17 -0700 Subject: [PATCH 007/177] fix workqueue --- src/core/lib/iomgr/ev_epollex_linux.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 608cda4a3bc..dd3c52ae321 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -278,9 +278,9 @@ static grpc_fd *fd_create(int fd, const char *name) { GRPC_LOG_IF_ERROR("fd_create", grpc_wakeup_fd_init(&new_fd->workqueue_wakeup_fd)); new_fd->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - new_fd->workqueue_read_mu = GPR_SPINLOCK_INIT; + new_fd->workqueue_read_mu = GPR_SPINLOCK_INITIALIZER; gpr_mpscq_init(&new_fd->workqueue_items); - gpr_atm_no_barrier_store(&new_fd->workqueue_item_count); + gpr_atm_no_barrier_store(&new_fd->workqueue_item_count, 0); new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -414,14 +414,14 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, GPR_TIMER_BEGIN("workqueue.enqueue", 0); grpc_fd *fd = (grpc_fd *)(((char *)closure->scheduler) - offsetof(grpc_fd, workqueue_scheduler)); - REF_BY(fd, 2); + REF_BY(fd, 2, "workqueue_enqueue"); gpr_atm last = gpr_atm_no_barrier_fetch_add(&fd->workqueue_item_count, 1); closure->error_data.error = error; gpr_mpscq_push(&fd->workqueue_items, &closure->next_data.atm_next); if (last == 0) { workqueue_wakeup(fd); } - UNREF_BY(fd, 2); + UNREF_BY(fd, 2, "workqueue_enqueue"); } static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { @@ -430,15 +430,14 @@ static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { gpr_mpscq_node *n = gpr_mpscq_pop(&fd->workqueue_items); gpr_spinlock_unlock(&fd->workqueue_read_mu); if (n != NULL) { - if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + if (gpr_atm_full_fetch_add(&fd->workqueue_item_count, -1) > 1) { workqueue_wakeup(fd); } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); - return true; - } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + } else if (gpr_atm_no_barrier_load(&fd->workqueue_item_count) > 0) { /* n == NULL might mean there's work but it's not available to be popped * yet - try to ensure another workqueue wakes up to check shortly if so */ From f4360d77d45b589f3f3820c1d3e23d9f1161846e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 08:51:00 -0700 Subject: [PATCH 008/177] Fix build, fix epollex detection --- CMakeLists.txt | 7 +++++ Makefile | 7 +++++ binding.gyp | 1 + build.yaml | 2 ++ config.m4 | 1 + gRPC-Core.podspec | 3 ++ grpc.gemspec | 2 ++ package.xml | 2 ++ src/core/lib/iomgr/ev_epollex_linux.c | 30 +++++++++++++++---- src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.c++.internal | 2 ++ tools/doxygen/Doxyfile.core.internal | 2 ++ .../generated/sources_and_headers.json | 3 ++ vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 ++ .../vcxproj/grpc++/grpc++.vcxproj.filters | 6 ++++ .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 ++ .../grpc++_unsecure.vcxproj.filters | 6 ++++ vsprojects/vcxproj/grpc/grpc.vcxproj | 3 ++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 ++++ .../grpc_test_util/grpc_test_util.vcxproj | 3 ++ .../grpc_test_util.vcxproj.filters | 6 ++++ .../grpc_unsecure/grpc_unsecure.vcxproj | 3 ++ .../grpc_unsecure.vcxproj.filters | 6 ++++ 23 files changed, 102 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca2b2598428..aa21ae6b2c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -933,6 +933,7 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1256,6 +1257,7 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1565,6 +1567,7 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1826,6 +1829,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2243,6 +2247,7 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2574,6 +2579,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -3273,6 +3279,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c diff --git a/Makefile b/Makefile index 1eeae6071eb..525a78937f1 100644 --- a/Makefile +++ b/Makefile @@ -2835,6 +2835,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3156,6 +3157,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3464,6 +3466,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3697,6 +3700,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4091,6 +4095,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4430,6 +4435,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -5121,6 +5127,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/binding.gyp b/binding.gyp index 3df6d0aa2f4..c1a9f733ea7 100644 --- a/binding.gyp +++ b/binding.gyp @@ -675,6 +675,7 @@ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/build.yaml b/build.yaml index c8098c51e5c..d258f91ac9b 100644 --- a/build.yaml +++ b/build.yaml @@ -199,6 +199,7 @@ filegroups: - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h - src/core/lib/iomgr/ev_epoll_linux.h + - src/core/lib/iomgr/ev_epollex_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -311,6 +312,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + - src/core/lib/iomgr/ev_epollex_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c - src/core/lib/iomgr/exec_ctx.c diff --git a/config.m4 b/config.m4 index 9470e43aaa0..e43c0296f8e 100644 --- a/config.m4 +++ b/config.m4 @@ -109,6 +109,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2e260189fbd..a84c12a999d 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -281,6 +281,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epollex_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -485,6 +486,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', @@ -733,6 +735,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epollex_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/grpc.gemspec b/grpc.gemspec index 1165a91b6f3..4054a49cc3c 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -197,6 +197,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epollex_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -401,6 +402,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epollex_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) diff --git a/package.xml b/package.xml index c6b86a66a2b..3910ade1bdf 100644 --- a/package.xml +++ b/package.xml @@ -206,6 +206,7 @@ + @@ -410,6 +411,7 @@ + diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index dd3c52ae321..5045697f600 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -328,6 +328,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; + if (!is_fd_closed) { + gpr_log(GPR_DEBUG, "TODO: handle fd removal?"); + } + /* Remove the active status but keep referenced. We want this grpc_fd struct to be alive (and not added to freelist) until the end of this function */ REF_BY(fd, 1, reason); @@ -858,13 +862,27 @@ static bool is_epollex_available(void) { grpc_wakeup_fd_init(&wakeup))) { return false; } - struct epoll_event ev = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = NULL}; + struct epoll_event ev = { + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + .events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT, + .data.ptr = NULL}; if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { + if (errno != EINVAL) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + close(fd); + grpc_wakeup_fd_destroy(&wakeup); + return false; + } + } else { gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE failed with error: %d. Not using " - "epollex polling engine", - fd); + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); close(fd); grpc_wakeup_fd_destroy(&wakeup); return false; @@ -897,7 +915,7 @@ const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 0d0a5fb0887..b74204337b7 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -103,6 +103,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index afab2296a02..b600de2fcb1 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -952,6 +952,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epollex_linux.c \ +src/core/lib/iomgr/ev_epollex_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ee1fd1b242f..97b0d8114e0 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1073,6 +1073,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epollex_linux.c \ +src/core/lib/iomgr/ev_epollex_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index fa62d0aaaca..6f5f1191314 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7540,6 +7540,7 @@ "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epollex_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -7692,6 +7693,8 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epollex_linux.c", + "src/core/lib/iomgr/ev_epollex_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index cae44c5e234..bf798a02a81 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -403,6 +403,7 @@ + @@ -628,6 +629,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 36da089470b..ee7447aa264 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -202,6 +202,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -944,6 +947,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 4d284006e67..2991de4f6fe 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -397,6 +397,7 @@ + @@ -612,6 +613,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 9d641553a2b..e3fb5cfe988 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -187,6 +187,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -911,6 +914,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index dd6fdd861e4..13f745c6da9 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -326,6 +326,7 @@ + @@ -559,6 +560,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 51c9b7201dc..7b02f2ce30f 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -82,6 +82,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -878,6 +881,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index b1db7e5d15d..3a87cdf311e 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -221,6 +221,7 @@ + @@ -400,6 +401,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index a7ecbea87c7..bead542aaa3 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -139,6 +139,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -656,6 +659,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 8614b115898..47aec42cce7 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -316,6 +316,7 @@ + @@ -527,6 +528,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index e7aa9fb8d6f..6ecbf24ed2d 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -85,6 +85,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -791,6 +794,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From ef814ca29ae27a1c9b8193b185c82bd958b68778 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 08:55:03 -0700 Subject: [PATCH 009/177] Sentences finish with a full stop --- src/core/lib/iomgr/ev_epollex_linux.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 5045697f600..e846263a4bf 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -851,10 +851,10 @@ static const grpc_event_engine_vtable vtable = { static bool is_epollex_available(void) { int fd = epoll_create1(EPOLL_CLOEXEC); if (fd < 0) { - gpr_log( - GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epollex polling engine", - fd); + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); return false; } grpc_wakeup_fd wakeup; From 8fc1ca1e3f3fe3cfb6040405f764a150a95110ba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 13:01:48 -0700 Subject: [PATCH 010/177] Initial pollset_set implementation --- src/core/lib/iomgr/ev_epollex_linux.c | 298 +++++++++++++++++++++++--- 1 file changed, 267 insertions(+), 31 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index e846263a4bf..6d79b82982d 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -73,13 +73,30 @@ static grpc_wakeup_fd global_wakeup_fd; /******************************************************************************* - * Fd Declarations + * Pollset-set sibling link */ -#define FD_FROM_PO(po) ((grpc_fd *)(po)) +typedef enum { + PSS_FD, + PSS_POLLSET, + PSS_POLLSET_SET, + PSS_OBJ_TYPE_COUNT +} pss_obj_type; -struct grpc_fd { +typedef struct pss_obj { gpr_mu mu; + struct pss_obj *pss_next; + struct pss_obj *pss_prev; + int pss_refs; + grpc_pollset_set *pss_master; +} pss_obj; + +/******************************************************************************* + * Fd Declarations + */ + +struct grpc_fd { + pss_obj po; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -137,19 +154,31 @@ struct grpc_pollset_worker { }; struct grpc_pollset { - gpr_mu mu; + pss_obj po; int epfd; int num_pollers; gpr_atm shutdown_atm; grpc_closure *shutdown_closure; grpc_wakeup_fd pollset_wakeup; grpc_pollset_worker *root_worker; + + grpc_pollset *pss_next; + grpc_pollset *pss_prev; + int pss_refs; + grpc_pollset_set *pss_master; }; /******************************************************************************* * Pollset-set Declarations */ -struct grpc_pollset_set {}; +struct grpc_pollset_set { + pss_obj po; + gpr_refcount refs; + grpc_pollset_set *master; + + /* roots are only used if master == self */ + pss_obj *roots[PSS_OBJ_TYPE_COUNT]; +}; /******************************************************************************* * Common helpers @@ -242,7 +271,7 @@ static void fd_global_shutdown(void) { while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->mu); + gpr_mu_destroy(&fd->po.mu); gpr_free(fd); } gpr_mu_destroy(&fd_freelist_mu); @@ -260,13 +289,13 @@ static grpc_fd *fd_create(int fd, const char *name) { if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->mu); + gpr_mu_init(&new_fd->po.mu); } - /* Note: It is not really needed to get the new_fd->mu lock here. If this + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this * is a newly created fd (or an fd we got from the freelist), no one else * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->mu); + gpr_mu_lock(&new_fd->po.mu); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -285,7 +314,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; - gpr_mu_unlock(&new_fd->mu); + gpr_mu_unlock(&new_fd->po.mu); char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); @@ -299,11 +328,11 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { int ret_fd = -1; - gpr_mu_lock(&fd->mu); + gpr_mu_lock(&fd->po.mu); if (!fd->orphaned) { ret_fd = fd->fd; } - gpr_mu_unlock(&fd->mu); + gpr_mu_unlock(&fd->po.mu); return ret_fd; } @@ -314,7 +343,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->mu); + gpr_mu_lock(&fd->po.mu); fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file @@ -338,7 +367,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); - gpr_mu_unlock(&fd->mu); + gpr_mu_unlock(&fd->po.mu); UNREF_BY(fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -472,7 +501,7 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } -/* p->mu must be held before calling this function */ +/* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker == NULL) { @@ -497,7 +526,7 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - gpr_mu_init(&pollset->mu); + gpr_mu_init(&pollset->po.mu); pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); @@ -523,7 +552,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { } } pollset->root_worker = NULL; - *mu = &pollset->mu; + *mu = &pollset->po.mu; } /* Convert a timespec to milliseconds: @@ -588,7 +617,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->mu); + gpr_mu_destroy(&pollset->po.mu); if (pollset->epfd >= 0) close(pollset->epfd); grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); } @@ -669,7 +698,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, worker->initialized_cv = true; gpr_cv_init(&worker->cv); while (pollset->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline)) return false; + if (gpr_cv_wait(&worker->cv, &pollset->po.mu, deadline)) return false; if (worker->kicked) return false; } } @@ -698,8 +727,8 @@ static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } } -/* pollset->mu lock must be held by the caller before calling this. - The function pollset_work() may temporarily release the lock (pollset->mu) +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) during the course of its execution but it will always re-acquire the lock and ensure that it is held by the time the function returns */ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -710,10 +739,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (begin_worker(pollset, &worker, worker_hdl, deadline)) { GPR_ASSERT(!pollset->shutdown_closure); pollset->num_pollers++; - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(&pollset->po.mu); error = pollset_poll(exec_ctx, pollset, now, deadline); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(&pollset->po.mu); pollset->num_pollers--; if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); @@ -758,45 +787,252 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, */ static grpc_pollset_set *pollset_set_create(void) { - grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); + gpr_mu_init(&pss->po.mu); + pss->roots[PSS_POLLSET_SET] = &pss->po; + pss->po.pss_next = pss->po.pss_prev = &pss->po; + return pss; +} + +static void pss_destroy(grpc_pollset_set *pss) { + gpr_mu_destroy(&pss->po.mu); + gpr_free(pss); +} + +static grpc_pollset_set *pss_ref(grpc_pollset_set *pss) { + gpr_ref(&pss->refs); return pss; } +static void pss_unref(grpc_pollset_set *pss) { + if (gpr_unref(&pss->refs)) pss_destroy(pss); +} + static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss) { - gpr_free(pss); + pss_unref(pss); +} + +static grpc_pollset_set *pss_ref_and_lock_master( + grpc_pollset_set *master_or_slave) { + pss_ref(master_or_slave); + gpr_mu_lock(&master_or_slave->po.mu); + while (master_or_slave != master_or_slave->master) { + grpc_pollset_set *master = pss_ref(master_or_slave->master); + gpr_mu_unlock(&master_or_slave->po.mu); + pss_unref(master_or_slave); + master_or_slave = master; + gpr_mu_lock(&master_or_slave->po.mu); + } + return master_or_slave; +} + +static void pss_broadcast_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *dst, + pss_obj *obj) { + grpc_fd *fd = (grpc_fd *)obj; + if (dst->roots[PSS_POLLSET] == NULL) return; + pss_obj *tgt = dst->roots[PSS_POLLSET]; + do { + pollset_add_fd(exec_ctx, (grpc_pollset *)tgt, fd); + tgt = tgt->pss_next; + } while (tgt != dst->roots[PSS_POLLSET]); +} + +static void pss_broadcast_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *dst, pss_obj *obj) { + grpc_pollset *pollset = (grpc_pollset *)obj; + if (dst->roots[PSS_FD] == NULL) return; + pss_obj *tgt = dst->roots[PSS_FD]; + do { + pollset_add_fd(exec_ctx, pollset, (grpc_fd *)tgt); + tgt = tgt->pss_next; + } while (tgt != dst->roots[PSS_FD]); +} + +static pss_obj *pss_splice(pss_obj *p, pss_obj *q) { + if (p == NULL) return q; + if (q == NULL) return p; + p->pss_next->pss_prev = q->pss_prev; + q->pss_prev->pss_next = p->pss_next; + p->pss_next = q; + q->pss_prev = p; + return p; +} + +static void (*const broadcast[PSS_OBJ_TYPE_COUNT])(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *dst, + pss_obj *obj) = { + pss_broadcast_fd, pss_broadcast_pollset, NULL}; + +static void pss_merge_broadcast_and_patch(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *a, + grpc_pollset_set *b, + pss_obj_type type) { + pss_obj *obj; + if (a->roots[type] != NULL) { + obj = a->roots[PSS_FD]; + do { + broadcast[type](exec_ctx, b, obj); + obj = obj->pss_next; + } while (obj != a->roots[PSS_FD]); + } + if (b->roots[type] != NULL) { + obj = b->roots[PSS_FD]; + do { + broadcast[type](exec_ctx, a, obj); + gpr_mu_lock(&obj->mu); + obj->pss_master = a; + gpr_mu_unlock(&obj->mu); + obj = obj->pss_next; + } while (obj != b->roots[PSS_FD]); + } + a->roots[type] = pss_splice(a->roots[type], b->roots[type]); +} + +static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, + grpc_pollset_set *b) { + pss_ref(a); + pss_ref(b); + bool changed; + for (;;) { + if (a == b) { + pss_unref(a); + pss_unref(b); + return; + } else if (a < b) { + gpr_mu_lock(&a->po.mu); + gpr_mu_lock(&b->po.mu); + } else { + gpr_mu_lock(&b->po.mu); + gpr_mu_lock(&a->po.mu); + } + changed = false; + if (a != a->master) { + grpc_pollset_set *master = pss_ref(a->master); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pss_unref(a); + a = master; + changed = true; + } else if (b != b->master) { + grpc_pollset_set *master = pss_ref(b->master); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pss_unref(b); + b = master; + changed = true; + } else { + /* a, b locked and are at their respective masters */ + pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); + pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); + b->po.pss_master = a; + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + } + } +} + +static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + pss_obj *obj, pss_obj_type type) { + pss = pss_ref_and_lock_master(pss); + gpr_mu_lock(&obj->mu); + if (obj->pss_master == pss) { + /* obj is already a member -- just bump refcount */ + obj->pss_refs++; + gpr_mu_unlock(&obj->mu); + gpr_mu_unlock(&pss->po.mu); + pss_unref(pss); + return; + } else if (obj->pss_master != NULL) { + grpc_pollset_set *other_pss = pss_ref(obj->pss_master); + obj->pss_refs++; + gpr_mu_unlock(&obj->mu); + gpr_mu_unlock(&pss->po.mu); + pss_merge(exec_ctx, pss, other_pss); + pss_unref(other_pss); + pss_unref(pss); + } else { + GPR_ASSERT(obj->pss_refs == 0); + obj->pss_refs = 1; + obj->pss_master = pss; + if (pss->roots[type] == NULL) { + pss->roots[type] = obj; + obj->pss_next = obj->pss_prev = obj; + } else { + obj->pss_next = pss->roots[type]; + obj->pss_prev = obj->pss_next->pss_prev; + obj->pss_prev->pss_next = obj; + obj->pss_next->pss_prev = obj; + } + gpr_mu_unlock(&obj->mu); + switch (type) { + case PSS_FD: + pss_broadcast_fd(exec_ctx, pss, obj); + break; + case PSS_POLLSET: + pss_broadcast_pollset(exec_ctx, pss, obj); + break; + case PSS_POLLSET_SET: + case PSS_OBJ_TYPE_COUNT: + GPR_UNREACHABLE_CODE(break); + } + gpr_mu_unlock(&pss->po.mu); + pss_unref(pss); + } +} + +static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + pss_obj *obj, pss_obj_type type) { + pss = pss_ref_and_lock_master(pss); + gpr_mu_lock(&obj->mu); + obj->pss_refs--; + if (obj->pss_refs == 0) { + obj->pss_master = NULL; + if (obj == pss->roots[type]) { + pss->roots[type] = obj->pss_next; + } + if (obj->pss_next == obj) { + pss->roots[type] = NULL; + } else { + obj->pss_next->pss_prev = obj->pss_prev; + obj->pss_prev->pss_next = obj->pss_next; + } + } + gpr_mu_unlock(&obj->mu); + gpr_mu_unlock(&pss->po.mu); + pss_unref(pss); } static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - abort(); + pss_add_obj(exec_ctx, pss, &fd->po, PSS_FD); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - abort(); + pss_del_obj(exec_ctx, pss, &fd->po, PSS_FD); } static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - abort(); + pss_add_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - abort(); + pss_del_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); } static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, grpc_pollset_set *item) { - abort(); + pss_add_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); } static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, grpc_pollset_set *item) { - abort(); + pss_del_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); } /******************************************************************************* From bcf8db50b90574dc83836990106b484e0f48a195 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 20:41:40 +0000 Subject: [PATCH 011/177] Fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 20 +++++++++----------- test/core/slice/slice_buffer_test.c | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 6d79b82982d..23ca26933a9 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -174,7 +174,6 @@ struct grpc_pollset { struct grpc_pollset_set { pss_obj po; gpr_refcount refs; - grpc_pollset_set *master; /* roots are only used if master == self */ pss_obj *roots[PSS_OBJ_TYPE_COUNT]; @@ -789,7 +788,9 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static grpc_pollset_set *pollset_set_create(void) { grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); gpr_mu_init(&pss->po.mu); + gpr_ref_init(&pss->refs, 1); pss->roots[PSS_POLLSET_SET] = &pss->po; + pss->po.pss_master = pss; pss->po.pss_next = pss->po.pss_prev = &pss->po; return pss; } @@ -817,8 +818,8 @@ static grpc_pollset_set *pss_ref_and_lock_master( grpc_pollset_set *master_or_slave) { pss_ref(master_or_slave); gpr_mu_lock(&master_or_slave->po.mu); - while (master_or_slave != master_or_slave->master) { - grpc_pollset_set *master = pss_ref(master_or_slave->master); + while (master_or_slave != master_or_slave->po.pss_master) { + grpc_pollset_set *master = pss_ref(master_or_slave->po.pss_master); gpr_mu_unlock(&master_or_slave->po.mu); pss_unref(master_or_slave); master_or_slave = master; @@ -893,7 +894,6 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, grpc_pollset_set *b) { pss_ref(a); pss_ref(b); - bool changed; for (;;) { if (a == b) { pss_unref(a); @@ -906,21 +906,18 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, gpr_mu_lock(&b->po.mu); gpr_mu_lock(&a->po.mu); } - changed = false; - if (a != a->master) { - grpc_pollset_set *master = pss_ref(a->master); + if (a != a->po.pss_master) { + grpc_pollset_set *master = pss_ref(a->po.pss_master); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); pss_unref(a); a = master; - changed = true; - } else if (b != b->master) { - grpc_pollset_set *master = pss_ref(b->master); + } else if (b != b->po.pss_master) { + grpc_pollset_set *master = pss_ref(b->po.pss_master); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); pss_unref(b); b = master; - changed = true; } else { /* a, b locked and are at their respective masters */ pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); @@ -928,6 +925,7 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, b->po.pss_master = a; gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); + return; } } } diff --git a/test/core/slice/slice_buffer_test.c b/test/core/slice/slice_buffer_test.c index bf9ae197d21..51cab2a87f4 100644 --- a/test/core/slice/slice_buffer_test.c +++ b/test/core/slice/slice_buffer_test.c @@ -115,8 +115,8 @@ void test_slice_buffer_move_first() { grpc_slice_buffer_move_first(&src, 2, &dst); src_len -= 2; dst_len += 2; - GPR_ASSERT(src.length == src.length); - GPR_ASSERT(dst.length == dst.length); + GPR_ASSERT(src_len == src.length); + GPR_ASSERT(dst_len == dst.length); } int main(int argc, char **argv) { From bd412f9d3d3a567568f7ba4f081447315c957495 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 13:45:19 -0700 Subject: [PATCH 012/177] always use pss_obj_init --- src/core/lib/iomgr/ev_epollex_linux.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 23ca26933a9..734f0c9a576 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -91,6 +91,14 @@ typedef struct pss_obj { grpc_pollset_set *pss_master; } pss_obj; +static void pss_obj_init(pss_obj *obj) { + gpr_mu_init(&obj->mu); + obj->pss_refs = 0; + obj->pss_next = NULL; + obj->pss_prev = NULL; + obj->pss_master = NULL; +} + /******************************************************************************* * Fd Declarations */ @@ -288,13 +296,9 @@ static grpc_fd *fd_create(int fd, const char *name) { if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->po.mu); } - /* Note: It is not really needed to get the new_fd->po.mu lock here. If this - * is a newly created fd (or an fd we got from the freelist), no one else - * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->po.mu); + pss_obj_init(&new_fd->po); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -313,8 +317,6 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; - gpr_mu_unlock(&new_fd->po.mu); - char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); @@ -525,7 +527,7 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - gpr_mu_init(&pollset->po.mu); + pss_obj_init(&pollset->po); pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); @@ -787,7 +789,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static grpc_pollset_set *pollset_set_create(void) { grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); - gpr_mu_init(&pss->po.mu); + pss_obj_init(&pss->po); gpr_ref_init(&pss->refs, 1); pss->roots[PSS_POLLSET_SET] = &pss->po; pss->po.pss_master = pss; From f081d8985a1a44da1fe35ba345e71bffd8d77945 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 14:04:42 -0700 Subject: [PATCH 013/177] different merge --- src/core/lib/iomgr/ev_epollex_linux.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 734f0c9a576..9bba20e9a60 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1026,14 +1026,12 @@ static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, grpc_pollset_set *item) { - pss_add_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); + pss_merge(exec_ctx, bag, item); } static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, grpc_pollset_set *bag, - grpc_pollset_set *item) { - pss_del_obj(exec_ctx, bag, &item->po, PSS_POLLSET_SET); -} + grpc_pollset_set *item) {} /******************************************************************************* * Event engine binding From 05da6e98c15f263d18c6d28e84e85b085784c490 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 21:21:37 +0000 Subject: [PATCH 014/177] Stash timeout in a stack var to ease debug --- src/core/lib/iomgr/ev_epollex_linux.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 9bba20e9a60..ac63bee95e3 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -404,7 +404,7 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { abort(); } +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { REF_BY(fd, 2, "return_workqueue"); return (grpc_workqueue*)fd; } #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, @@ -636,8 +636,9 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } GRPC_SCHEDULING_START_BLOCKING_REGION; +int timeout=poll_deadline_to_millis_timeout(deadline, now); int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, - poll_deadline_to_millis_timeout(deadline, now)); + timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); @@ -873,21 +874,21 @@ static void pss_merge_broadcast_and_patch(grpc_exec_ctx *exec_ctx, pss_obj_type type) { pss_obj *obj; if (a->roots[type] != NULL) { - obj = a->roots[PSS_FD]; + obj = a->roots[type]; do { broadcast[type](exec_ctx, b, obj); obj = obj->pss_next; - } while (obj != a->roots[PSS_FD]); + } while (obj != a->roots[type]); } if (b->roots[type] != NULL) { - obj = b->roots[PSS_FD]; + obj = b->roots[type]; do { broadcast[type](exec_ctx, a, obj); gpr_mu_lock(&obj->mu); obj->pss_master = a; gpr_mu_unlock(&obj->mu); obj = obj->pss_next; - } while (obj != b->roots[PSS_FD]); + } while (obj != b->roots[type]); } a->roots[type] = pss_splice(a->roots[type], b->roots[type]); } From 7f5fac90547d54f0f98d510995e697e6ea3ec9ed Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 21:47:41 +0000 Subject: [PATCH 015/177] minor optimizations --- src/core/lib/iomgr/ev_epollex_linux.c | 10 ++++------ test/cpp/microbenchmarks/bm_pollset.cc | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index ac63bee95e3..cb996d8884c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -566,19 +566,17 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { gpr_timespec timeout; - static const int64_t max_spin_polling_us = 10; if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { return -1; } - if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( - max_spin_polling_us, - GPR_TIMESPAN))) <= 0) { + if (gpr_time_cmp(deadline, now) <= 0) { return 0; } + + static const gpr_timespec round_up = {.clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS-1}; timeout = gpr_time_sub(deadline, now); - int millis = gpr_time_to_millis(gpr_time_add( - timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; } diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 0f3d3cef66a..3789deebde1 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -136,8 +136,7 @@ static void BM_PollEmptyPollset(benchmark::State& state) { gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); gpr_mu_lock(mu); while (state.KeepRunning()) { - grpc_pollset_worker* worker; - GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, NULL, now, deadline)); } grpc_closure shutdown_ps_closure; grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, @@ -233,8 +232,7 @@ static void BM_SingleThreadPollOneFd(benchmark::State& state) { grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure); gpr_mu_lock(mu); while (!done) { - grpc_pollset_worker* worker; - GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline)); + GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, NULL, now, deadline)); } grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done"); wakeup_fd.read_fd = 0; From eb73f80085cd85224b7811886b3b2805c920dbc5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 21:55:34 +0000 Subject: [PATCH 016/177] Add epollex polling engine to run_tests.py --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0b4f26ca440..18267888596 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -72,7 +72,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { _POLLING_STRATEGIES = { - 'linux': ['epoll', 'poll', 'poll-cv'] + 'linux': ['epollex', 'epoll', 'poll', 'poll-cv'] } From cd4a214d8a0475d22147260dfd736cb7a915921b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 22:14:58 +0000 Subject: [PATCH 017/177] fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index cb996d8884c..c75b59c4c40 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -796,8 +796,16 @@ static grpc_pollset_set *pollset_set_create(void) { return pss; } +static void pss_unref(grpc_pollset_set *pss); + static void pss_destroy(grpc_pollset_set *pss) { gpr_mu_destroy(&pss->po.mu); + GPR_ASSERT(pss->roots[PSS_FD] == NULL); + GPR_ASSERT(pss->roots[PSS_POLLSET] == NULL); + GPR_ASSERT(pss->roots[PSS_POLLSET_SET] == &pss->po); + for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; child = child->pss_next) { + pss_unref((grpc_pollset_set*)child); + } gpr_free(pss); } @@ -924,8 +932,11 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); b->po.pss_master = a; + a->roots[PSS_POLLSET_SET] = pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); + pss_unref(a); + /* a now owns a ref to b */ return; } } From 8c6878be3a5f748ccbc68be261303a3b57837854 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 15:15:30 -0700 Subject: [PATCH 018/177] clang-format --- src/core/lib/iomgr/ev_epollex_linux.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c75b59c4c40..89b416a60e1 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -404,7 +404,10 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { REF_BY(fd, 2, "return_workqueue"); return (grpc_workqueue*)fd; } +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + REF_BY(fd, 2, "return_workqueue"); + return (grpc_workqueue *)fd; +} #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, @@ -574,7 +577,8 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline, return 0; } - static const gpr_timespec round_up = {.clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS-1}; + static const gpr_timespec round_up = { + .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; timeout = gpr_time_sub(deadline, now); int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); return millis >= 1 ? millis : 1; @@ -634,9 +638,8 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } GRPC_SCHEDULING_START_BLOCKING_REGION; -int timeout=poll_deadline_to_millis_timeout(deadline, now); - int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, - timeout); + int timeout = poll_deadline_to_millis_timeout(deadline, now); + int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); @@ -803,8 +806,9 @@ static void pss_destroy(grpc_pollset_set *pss) { GPR_ASSERT(pss->roots[PSS_FD] == NULL); GPR_ASSERT(pss->roots[PSS_POLLSET] == NULL); GPR_ASSERT(pss->roots[PSS_POLLSET_SET] == &pss->po); - for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; child = child->pss_next) { - pss_unref((grpc_pollset_set*)child); + for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; + child = child->pss_next) { + pss_unref((grpc_pollset_set *)child); } gpr_free(pss); } @@ -932,7 +936,8 @@ static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); b->po.pss_master = a; - a->roots[PSS_POLLSET_SET] = pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); + a->roots[PSS_POLLSET_SET] = + pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); gpr_mu_unlock(&a->po.mu); gpr_mu_unlock(&b->po.mu); pss_unref(a); From 2636f43a6789f63148448fdce2877aff6b0aaade Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 15:16:56 -0700 Subject: [PATCH 019/177] less spam --- src/core/lib/iomgr/ev_epollex_linux.c | 36 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 89b416a60e1..c71b1fe8538 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1098,12 +1098,17 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ static bool is_epollex_available(void) { + static bool logged_why_not = false; + int fd = epoll_create1(EPOLL_CLOEXEC); if (fd < 0) { - gpr_log(GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epollex polling " - "engine.", - fd); + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } return false; } grpc_wakeup_fd wakeup; @@ -1119,19 +1124,26 @@ static bool is_epollex_available(void) { .data.ptr = NULL}; if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { if (errno != EINVAL) { - gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " - "%d. Not using epollex polling engine.", - errno); + if (!logged_why_not) { + gpr_log( + GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + logged_why_not = true; + } close(fd); grpc_wakeup_fd_destroy(&wakeup); return false; } } else { - gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " - "evidence of no EPOLLEXCLUSIVE support. Not using " - "epollex polling engine."); + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); + logged_why_not = true; + } close(fd); grpc_wakeup_fd_destroy(&wakeup); return false; From d9cd8f0abe6aa8562aaafb041b1db371247517d6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 16:26:02 -0700 Subject: [PATCH 020/177] shutdown progress --- src/core/lib/iomgr/ev_epollex_linux.c | 32 ++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c71b1fe8538..172847f6c84 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -601,11 +601,20 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } +static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0 && + pollset->po.pss_master == NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + } +} + /* pollset->po.mu lock must be held by the caller before calling this */ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { GPR_ASSERT(pollset->shutdown_closure == NULL); pollset->shutdown_closure = closure; + gpr_atm_no_barrier_store(&pollset->shutdown_atm, 1); if (pollset->num_pollers > 0) { struct epoll_event ev = {.events = EPOLLIN, .data.ptr = &pollset->pollset_wakeup}; @@ -613,9 +622,16 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, &ev); GRPC_LOG_IF_ERROR("pollset_shutdown", grpc_wakeup_fd_wakeup(&pollset->pollset_wakeup)); - } else { - grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } + if (pollset->root_worker != NULL) { + for (grpc_pollset_worker *worker = pollset->root_worker->next; + worker != pollset->root_worker; worker = worker->next) { + if (worker->initialized_cv) { + gpr_cv_signal(&worker->cv); + } + } + } + pollset_maybe_finish_shutdown(exec_ctx, pollset); } /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ @@ -719,6 +735,7 @@ static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, pollset->root_worker = worker->next; worker->prev->next = worker->next; worker->next->prev = worker->prev; + gpr_cv_signal(&pollset->root_worker->cv); } } else { worker->prev->next = worker->next; @@ -747,9 +764,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->po.mu); pollset->num_pollers--; - if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { - grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); - } + pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(pollset, &worker, worker_hdl); return error; @@ -979,12 +994,14 @@ static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, obj->pss_prev->pss_next = obj; obj->pss_next->pss_prev = obj; } - gpr_mu_unlock(&obj->mu); switch (type) { case PSS_FD: + REF_BY((grpc_fd *)obj, 2, "pollset_set"); + gpr_mu_unlock(&obj->mu); pss_broadcast_fd(exec_ctx, pss, obj); break; case PSS_POLLSET: + gpr_mu_unlock(&obj->mu); pss_broadcast_pollset(exec_ctx, pss, obj); break; case PSS_POLLSET_SET: @@ -998,6 +1015,7 @@ static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, pss_obj *obj, pss_obj_type type) { + bool unref = false; pss = pss_ref_and_lock_master(pss); gpr_mu_lock(&obj->mu); obj->pss_refs--; @@ -1012,10 +1030,12 @@ static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, obj->pss_next->pss_prev = obj->pss_prev; obj->pss_prev->pss_next = obj->pss_next; } + unref = true; } gpr_mu_unlock(&obj->mu); gpr_mu_unlock(&pss->po.mu); pss_unref(pss); + if (unref && type == PSS_FD) UNREF_BY((grpc_fd *)obj, 2, "pollset_set"); } static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, From bb93af6b86d0a097db109ab0aa0b0f7a43cb2a3e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 23:49:00 +0000 Subject: [PATCH 021/177] kick optimizations --- src/core/lib/iomgr/ev_epollex_linux.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c75b59c4c40..d3cc1d1952a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -165,6 +165,7 @@ struct grpc_pollset { pss_obj po; int epfd; int num_pollers; + bool kicked_without_poller; gpr_atm shutdown_atm; grpc_closure *shutdown_closure; grpc_wakeup_fd pollset_wakeup; @@ -507,7 +508,12 @@ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { - return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + if (p->num_pollers == 0) { + p->kicked_without_poller = true; + return GRPC_ERROR_NONE; + } else { + return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + } } else { return GRPC_ERROR_NONE; } @@ -528,6 +534,7 @@ static grpc_error *kick_poller(void) { static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pss_obj_init(&pollset->po); + pollset->kicked_without_poller = false; pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); @@ -736,13 +743,21 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; grpc_error *error = GRPC_ERROR_NONE; + if (pollset->kicked_without_poller) { + pollset->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } if (begin_worker(pollset, &worker, worker_hdl, deadline)) { + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); pollset->num_pollers++; gpr_mu_unlock(&pollset->po.mu); error = pollset_poll(exec_ctx, pollset, now, deadline); grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->po.mu); + gpr_tls_set(&g_current_thread_pollset, 0); + gpr_tls_set(&g_current_thread_worker, 0); pollset->num_pollers--; if (pollset->num_pollers == 0 && pollset->shutdown_closure != NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); From f18286bda6a7a117f5e8e33c6d65446c141d150f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 14:44:09 -0700 Subject: [PATCH 022/177] attempt #2 at pollset_set --- src/core/lib/iomgr/ev_epollex_linux.c | 508 +++++++++++++------------- 1 file changed, 253 insertions(+), 255 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index de780ec4172..958f4d50280 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -77,34 +77,48 @@ static grpc_wakeup_fd global_wakeup_fd; */ typedef enum { - PSS_FD, - PSS_POLLSET, - PSS_POLLSET_SET, - PSS_OBJ_TYPE_COUNT -} pss_obj_type; + PO_FD, + PO_POLLSET, + PO_POLLSET_SET, + PO_POLLING_GROUP, + PO_COUNT +} polling_obj_type; -typedef struct pss_obj { +typedef struct polling_obj polling_obj; +typedef struct polling_group polling_group; + +struct polling_obj { gpr_mu mu; - struct pss_obj *pss_next; - struct pss_obj *pss_prev; - int pss_refs; - grpc_pollset_set *pss_master; -} pss_obj; - -static void pss_obj_init(pss_obj *obj) { - gpr_mu_init(&obj->mu); - obj->pss_refs = 0; - obj->pss_next = NULL; - obj->pss_prev = NULL; - obj->pss_master = NULL; -} + polling_obj_type type; + polling_group *group; + struct polling_obj *next; + struct polling_obj *prev; +}; + +struct polling_group { + polling_obj po; + gpr_refcount refs; +}; + +static void po_init(polling_obj *po, polling_obj_type type); +static void po_destroy(polling_obj *po); +static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b); + +static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, + size_t initial_po_count); +static polling_group *pg_ref(polling_group *pg); +static void pg_unref(polling_group *pg); +static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, + polling_group *b); +static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, + polling_obj *po); /******************************************************************************* * Fd Declarations */ struct grpc_fd { - pss_obj po; + polling_obj po; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -162,7 +176,7 @@ struct grpc_pollset_worker { }; struct grpc_pollset { - pss_obj po; + polling_obj po; int epfd; int num_pollers; bool kicked_without_poller; @@ -181,11 +195,7 @@ struct grpc_pollset { * Pollset-set Declarations */ struct grpc_pollset_set { - pss_obj po; - gpr_refcount refs; - - /* roots are only used if master == self */ - pss_obj *roots[PSS_OBJ_TYPE_COUNT]; + polling_obj po; }; /******************************************************************************* @@ -257,10 +267,11 @@ static void unref_by(grpc_fd *fd, int n) { old = gpr_atm_full_fetch_add(&fd->refst, -n); if (old == n) { /* Add the fd to the freelist */ + grpc_iomgr_unregister_object(&fd->iomgr_object); + po_destroy(&fd->po); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); grpc_lfev_destroy(&fd->read_closure); grpc_lfev_destroy(&fd->write_closure); @@ -279,7 +290,6 @@ static void fd_global_shutdown(void) { while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->po.mu); gpr_free(fd); } gpr_mu_destroy(&fd_freelist_mu); @@ -299,7 +309,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = gpr_malloc(sizeof(grpc_fd)); } - pss_obj_init(&new_fd->po); + po_init(&new_fd->po, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -536,7 +546,7 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - pss_obj_init(&pollset->po); + po_init(&pollset->po, PO_POLLSET); pollset->kicked_without_poller = false; pollset->epfd = epoll_create1(EPOLL_CLOEXEC); if (pollset->epfd < 0) { @@ -610,8 +620,7 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0 && - pollset->po.pss_master == NULL) { + if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); } } @@ -643,7 +652,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->po.mu); + po_destroy(&pollset->po); if (pollset->epfd >= 0) close(pollset->epfd); grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); } @@ -821,268 +830,257 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static grpc_pollset_set *pollset_set_create(void) { grpc_pollset_set *pss = gpr_zalloc(sizeof(*pss)); - pss_obj_init(&pss->po); - gpr_ref_init(&pss->refs, 1); - pss->roots[PSS_POLLSET_SET] = &pss->po; - pss->po.pss_master = pss; - pss->po.pss_next = pss->po.pss_prev = &pss->po; + po_init(&pss->po, PO_POLLSET_SET); return pss; } -static void pss_unref(grpc_pollset_set *pss); - -static void pss_destroy(grpc_pollset_set *pss) { - gpr_mu_destroy(&pss->po.mu); - GPR_ASSERT(pss->roots[PSS_FD] == NULL); - GPR_ASSERT(pss->roots[PSS_POLLSET] == NULL); - GPR_ASSERT(pss->roots[PSS_POLLSET_SET] == &pss->po); - for (pss_obj *child = pss->roots[PSS_POLLSET_SET]; child != &pss->po; - child = child->pss_next) { - pss_unref((grpc_pollset_set *)child); - } - gpr_free(pss); +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + po_destroy(&pss->po); } -static grpc_pollset_set *pss_ref(grpc_pollset_set *pss) { - gpr_ref(&pss->refs); - return pss; +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + po_join(exec_ctx, &pss->po, &fd->po); } -static void pss_unref(grpc_pollset_set *pss) { - if (gpr_unref(&pss->refs)) pss_destroy(pss); -} +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} -static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss) { - pss_unref(pss); +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + po_join(exec_ctx, &pss->po, &ps->po); } -static grpc_pollset_set *pss_ref_and_lock_master( - grpc_pollset_set *master_or_slave) { - pss_ref(master_or_slave); - gpr_mu_lock(&master_or_slave->po.mu); - while (master_or_slave != master_or_slave->po.pss_master) { - grpc_pollset_set *master = pss_ref(master_or_slave->po.pss_master); - gpr_mu_unlock(&master_or_slave->po.mu); - pss_unref(master_or_slave); - master_or_slave = master; - gpr_mu_lock(&master_or_slave->po.mu); - } - return master_or_slave; -} +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} -static void pss_broadcast_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *dst, - pss_obj *obj) { - grpc_fd *fd = (grpc_fd *)obj; - if (dst->roots[PSS_POLLSET] == NULL) return; - pss_obj *tgt = dst->roots[PSS_POLLSET]; - do { - pollset_add_fd(exec_ctx, (grpc_pollset *)tgt, fd); - tgt = tgt->pss_next; - } while (tgt != dst->roots[PSS_POLLSET]); +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + po_join(exec_ctx, &bag->po, &item->po); } -static void pss_broadcast_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *dst, pss_obj *obj) { - grpc_pollset *pollset = (grpc_pollset *)obj; - if (dst->roots[PSS_FD] == NULL) return; - pss_obj *tgt = dst->roots[PSS_FD]; - do { - pollset_add_fd(exec_ctx, pollset, (grpc_fd *)tgt); - tgt = tgt->pss_next; - } while (tgt != dst->roots[PSS_FD]); -} +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} -static pss_obj *pss_splice(pss_obj *p, pss_obj *q) { - if (p == NULL) return q; - if (q == NULL) return p; - p->pss_next->pss_prev = q->pss_prev; - q->pss_prev->pss_next = p->pss_next; - p->pss_next = q; - q->pss_prev = p; - return p; +static void po_init(polling_obj *po, polling_obj_type type) { + gpr_mu_init(&po->mu); + po->type = type; + po->group = NULL; + po->next = po; + po->prev = po; } -static void (*const broadcast[PSS_OBJ_TYPE_COUNT])(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *dst, - pss_obj *obj) = { - pss_broadcast_fd, pss_broadcast_pollset, NULL}; - -static void pss_merge_broadcast_and_patch(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *a, - grpc_pollset_set *b, - pss_obj_type type) { - pss_obj *obj; - if (a->roots[type] != NULL) { - obj = a->roots[type]; - do { - broadcast[type](exec_ctx, b, obj); - obj = obj->pss_next; - } while (obj != a->roots[type]); - } - if (b->roots[type] != NULL) { - obj = b->roots[type]; - do { - broadcast[type](exec_ctx, a, obj); - gpr_mu_lock(&obj->mu); - obj->pss_master = a; - gpr_mu_unlock(&obj->mu); - obj = obj->pss_next; - } while (obj != b->roots[type]); +static polling_group *pg_lock_latest(polling_group *pg) { + /* assumes pg unlocked; consumes ref, returns ref */ + gpr_mu_lock(&pg->po.mu); + while (pg->po.group != NULL) { + polling_group *new_pg = pg_ref(pg->po.group); + gpr_mu_unlock(&pg->po.mu); + pg_unref(pg); + pg = new_pg; + gpr_mu_lock(&pg->po.mu); } - a->roots[type] = pss_splice(a->roots[type], b->roots[type]); + return pg; } -static void pss_merge(grpc_exec_ctx *exec_ctx, grpc_pollset_set *a, - grpc_pollset_set *b) { - pss_ref(a); - pss_ref(b); - for (;;) { - if (a == b) { - pss_unref(a); - pss_unref(b); - return; - } else if (a < b) { - gpr_mu_lock(&a->po.mu); - gpr_mu_lock(&b->po.mu); - } else { - gpr_mu_lock(&b->po.mu); - gpr_mu_lock(&a->po.mu); - } - if (a != a->po.pss_master) { - grpc_pollset_set *master = pss_ref(a->po.pss_master); - gpr_mu_unlock(&a->po.mu); - gpr_mu_unlock(&b->po.mu); - pss_unref(a); - a = master; - } else if (b != b->po.pss_master) { - grpc_pollset_set *master = pss_ref(b->po.pss_master); - gpr_mu_unlock(&a->po.mu); - gpr_mu_unlock(&b->po.mu); - pss_unref(b); - b = master; - } else { - /* a, b locked and are at their respective masters */ - pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_FD); - pss_merge_broadcast_and_patch(exec_ctx, a, b, PSS_POLLSET); - b->po.pss_master = a; - a->roots[PSS_POLLSET_SET] = - pss_splice(a->roots[PSS_POLLSET_SET], b->roots[PSS_POLLSET_SET]); - gpr_mu_unlock(&a->po.mu); - gpr_mu_unlock(&b->po.mu); - pss_unref(a); - /* a now owns a ref to b */ - return; - } +static void po_destroy(polling_obj *po) { + if (po->group != NULL) { + polling_group *pg = pg_lock_latest(po->group); + po->prev->next = po->next; + po->next->prev = po->prev; + pg_unref(pg); } + gpr_mu_destroy(&po->mu); } -static void pss_add_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - pss_obj *obj, pss_obj_type type) { - pss = pss_ref_and_lock_master(pss); - gpr_mu_lock(&obj->mu); - if (obj->pss_master == pss) { - /* obj is already a member -- just bump refcount */ - obj->pss_refs++; - gpr_mu_unlock(&obj->mu); - gpr_mu_unlock(&pss->po.mu); - pss_unref(pss); - return; - } else if (obj->pss_master != NULL) { - grpc_pollset_set *other_pss = pss_ref(obj->pss_master); - obj->pss_refs++; - gpr_mu_unlock(&obj->mu); - gpr_mu_unlock(&pss->po.mu); - pss_merge(exec_ctx, pss, other_pss); - pss_unref(other_pss); - pss_unref(pss); - } else { - GPR_ASSERT(obj->pss_refs == 0); - obj->pss_refs = 1; - obj->pss_master = pss; - if (pss->roots[type] == NULL) { - pss->roots[type] = obj; - obj->pss_next = obj->pss_prev = obj; - } else { - obj->pss_next = pss->roots[type]; - obj->pss_prev = obj->pss_next->pss_prev; - obj->pss_prev->pss_next = obj; - obj->pss_next->pss_prev = obj; - } - switch (type) { - case PSS_FD: - REF_BY((grpc_fd *)obj, 2, "pollset_set"); - gpr_mu_unlock(&obj->mu); - pss_broadcast_fd(exec_ctx, pss, obj); - break; - case PSS_POLLSET: - gpr_mu_unlock(&obj->mu); - pss_broadcast_pollset(exec_ctx, pss, obj); - break; - case PSS_POLLSET_SET: - case PSS_OBJ_TYPE_COUNT: - GPR_UNREACHABLE_CODE(break); - } - gpr_mu_unlock(&pss->po.mu); - pss_unref(pss); +static polling_group *pg_ref(polling_group *pg) { + gpr_ref(&pg->refs); + return pg; +} + +static void pg_unref(polling_group *pg) { + if (gpr_unref(&pg->refs)) { + po_destroy(&pg->po); + gpr_free(pg); } } -static void pss_del_obj(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - pss_obj *obj, pss_obj_type type) { - bool unref = false; - pss = pss_ref_and_lock_master(pss); - gpr_mu_lock(&obj->mu); - obj->pss_refs--; - if (obj->pss_refs == 0) { - obj->pss_master = NULL; - if (obj == pss->roots[type]) { - pss->roots[type] = obj->pss_next; - } - if (obj->pss_next == obj) { - pss->roots[type] = NULL; +static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { + if (a == b) return; + if (a > b) GPR_SWAP(polling_obj *, a, b); + + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + + if (a->group == NULL) { + if (b->group == NULL) { + polling_obj *initial_po[] = {a, b}; + pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); } else { - obj->pss_next->pss_prev = obj->pss_prev; - obj->pss_prev->pss_next = obj->pss_next; + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + pg_join(exec_ctx, b_group, a); } - unref = true; + } else if (b->group == NULL) { + polling_group *a_group = pg_ref(a->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_join(exec_ctx, a_group, b); + } else if (a->group == b->group) { + /* nothing to do */ + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *a_group = pg_ref(a->group); + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_merge(exec_ctx, a_group, b_group); } - gpr_mu_unlock(&obj->mu); - gpr_mu_unlock(&pss->po.mu); - pss_unref(pss); - if (unref && type == PSS_FD) UNREF_BY((grpc_fd *)obj, 2, "pollset_set"); } -static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - grpc_fd *fd) { - pss_add_obj(exec_ctx, pss, &fd->po, PSS_FD); +#define POTYPES(a, b) (((a)*PO_COUNT) + (b)) + +static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { + GPR_ASSERT(a != b); + switch (POTYPES(a->type, b->type)) { + case POTYPES(PO_FD, PO_POLLSET): + pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); + break; + case POTYPES(PO_POLLSET, PO_FD): + pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); + break; + } } -static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, - grpc_fd *fd) { - pss_del_obj(exec_ctx, pss, &fd->po, PSS_FD); +static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, + polling_group *to) { + for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { + for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { + pg_notify(exec_ctx, a, b); + } + } } -static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss, grpc_pollset *ps) { - pss_add_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); +static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, + size_t initial_po_count) { + /* assumes all polling objects in initial_po are locked */ + polling_group *pg = gpr_malloc(sizeof(*pg)); + po_init(&pg->po, PO_POLLING_GROUP); + gpr_ref_init(&pg->refs, initial_po_count); + GPR_ASSERT(initial_po[0]->group == NULL); + initial_po[0]->next = initial_po[0]->prev = initial_po[0]; + initial_po[0]->group = pg; + for (size_t i = 0; i < initial_po_count; i++) { + GPR_ASSERT(initial_po[i]->group == NULL); + initial_po[i]->group = pg; + } + for (size_t i = 1; i < initial_po_count; i++) { + initial_po[i]->prev = initial_po[i - 1]; + } + for (size_t i = 0; i < initial_po_count - 1; i++) { + initial_po[i]->next = initial_po[i + 1]; + } + initial_po[0]->prev = &pg->po; + initial_po[initial_po_count - 1]->next = &pg->po; + pg->po.next = initial_po[0]; + pg->po.prev = initial_po[initial_po_count - 1]; + for (size_t i = 1; i < initial_po_count; i++) { + for (size_t j = 0; j < i; j++) { + pg_notify(exec_ctx, initial_po[i], initial_po[j]); + } + } } -static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *pss, grpc_pollset *ps) { - pss_del_obj(exec_ctx, pss, &ps->po, PSS_POLLSET); +static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, + polling_obj *po) { + /* assumes neither pg nor po are locked; consumes one ref to pg */ + pg = pg_lock_latest(pg); + /* pg locked */ + gpr_mu_lock(&po->mu); + if (po->group != NULL) { + gpr_mu_unlock(&pg->po.mu); + polling_group *po_group = pg_ref(po->group); + gpr_mu_unlock(&po->mu); + pg_merge(exec_ctx, pg, po_group); + /* early exit: polling obj picked up a group before joining: we now need + to do a full merge */ + return; + } + /* pg, po locked */ + for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; + existing != &pg->po; existing = existing->next) { + pg_notify(exec_ctx, po, existing); + } + po->group = pg; + po->next = &pg->po; + po->prev = pg->po.prev; + po->prev->next = po->next->prev = po; + gpr_mu_unlock(&pg->po.mu); + gpr_mu_unlock(&po->mu); } -static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) { - pss_merge(exec_ctx, bag, item); +static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, + polling_group *b) { + for (;;) { + if (a == b) return; + if (a > b) GPR_SWAP(polling_group *, a, b); + gpr_mu_lock(&a->po.mu); + gpr_mu_lock(&b->po.mu); + if (a->po.group != NULL) { + polling_group *m2 = pg_ref(a->po.group); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pg_unref(a); + a = m2; + } else if (b->po.group != NULL) { + polling_group *m2 = pg_ref(b->po.group); + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + pg_unref(b); + b = m2; + } else { + break; + } + } + polling_group **unref = NULL; + size_t unref_count = 0; + size_t unref_cap = 0; + b->po.group = a; + pg_broadcast(exec_ctx, a, b); + pg_broadcast(exec_ctx, b, a); + while (b->po.next != &b->po) { + polling_obj *po = b->po.next; + gpr_mu_lock(&po->mu); + if (unref_count == unref_cap) { + unref_cap = GPR_MAX(8, 3 * unref_cap / 2); + unref = gpr_realloc(unref, unref_cap * sizeof(*unref)); + } + unref[unref_count++] = po->group; + po->group = pg_ref(a); + // unlink from b + po->prev->next = po->next; + po->next->prev = po->prev; + // link to a + po->next = &a->po; + po->prev = a->po.prev; + po->next->prev = po->prev->next = po; + gpr_mu_unlock(&po->mu); + } + gpr_mu_unlock(&a->po.mu); + gpr_mu_unlock(&b->po.mu); + for (size_t i = 0; i < unref_count; i++) { + pg_unref(unref[i]); + } + gpr_free(unref); } -static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, - grpc_pollset_set *bag, - grpc_pollset_set *item) {} - /******************************************************************************* * Event engine binding */ From ac328d2981ba0730ff4656106463282da4cbda80 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 14:45:15 -0700 Subject: [PATCH 023/177] fix compile --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 958f4d50280..d353eaf0426 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -973,7 +973,7 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, /* assumes all polling objects in initial_po are locked */ polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); - gpr_ref_init(&pg->refs, initial_po_count); + gpr_ref_init(&pg->refs, (int)initial_po_count); GPR_ASSERT(initial_po[0]->group == NULL); initial_po[0]->next = initial_po[0]->prev = initial_po[0]; initial_po[0]->group = pg; From 105016680a8b50ad48182bf94e449db6d3a85bcf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 21:47:08 +0000 Subject: [PATCH 024/177] refine code --- src/core/lib/iomgr/ev_epollex_linux.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d353eaf0426..4dbc8b127c1 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -974,9 +974,6 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); gpr_ref_init(&pg->refs, (int)initial_po_count); - GPR_ASSERT(initial_po[0]->group == NULL); - initial_po[0]->next = initial_po[0]->prev = initial_po[0]; - initial_po[0]->group = pg; for (size_t i = 0; i < initial_po_count; i++) { GPR_ASSERT(initial_po[i]->group == NULL); initial_po[i]->group = pg; From 3e21ec5c5ec4629c66078a44beb02c70aac1e5f9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 15:11:38 -0700 Subject: [PATCH 025/177] add missing unlock --- src/core/lib/iomgr/ev_epollex_linux.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4dbc8b127c1..3330d86d854 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -891,6 +891,7 @@ static void po_destroy(polling_obj *po) { polling_group *pg = pg_lock_latest(po->group); po->prev->next = po->next; po->next->prev = po->prev; + gpr_mu_unlock(&pg->po.mu); pg_unref(pg); } gpr_mu_destroy(&po->mu); @@ -974,6 +975,9 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); gpr_ref_init(&pg->refs, (int)initial_po_count); + GPR_ASSERT(initial_po[0]->group == NULL); + initial_po[0]->next = initial_po[0]->prev = initial_po[0]; + initial_po[0]->group = pg; for (size_t i = 0; i < initial_po_count; i++) { GPR_ASSERT(initial_po[i]->group == NULL); initial_po[i]->group = pg; From 9b59114f73cc081716669e3866eb058ad69a69c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 22:12:44 +0000 Subject: [PATCH 026/177] huh --- src/core/lib/iomgr/ev_epollex_linux.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 3330d86d854..b4c879e4314 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -975,9 +975,6 @@ static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, polling_group *pg = gpr_malloc(sizeof(*pg)); po_init(&pg->po, PO_POLLING_GROUP); gpr_ref_init(&pg->refs, (int)initial_po_count); - GPR_ASSERT(initial_po[0]->group == NULL); - initial_po[0]->next = initial_po[0]->prev = initial_po[0]; - initial_po[0]->group = pg; for (size_t i = 0; i < initial_po_count; i++) { GPR_ASSERT(initial_po[i]->group == NULL); initial_po[i]->group = pg; From 83d5fb6e61a8f818c41929a81984b831244c3b67 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 22:13:43 +0000 Subject: [PATCH 027/177] fix memory leak --- src/core/lib/iomgr/ev_epollex_linux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b4c879e4314..c6505f6868e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -837,6 +837,7 @@ static grpc_pollset_set *pollset_set_create(void) { static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss) { po_destroy(&pss->po); + gpr_free(pss); } static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, From 59b86dd7957be126d542c9f46c24ce8b9a80bd98 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 23:11:08 +0000 Subject: [PATCH 028/177] debug --- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index c6505f6868e..2afc20daa53 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -519,6 +519,7 @@ static void pollset_global_shutdown(void) { /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { + gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d root_worker=%p", p, specific_worker, (void*)gpr_tls_get(&g_current_thread_pollset), (void*)gpr_tls_get(&g_current_thread_worker), p->num_pollers, p->root_worker); if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { if (p->num_pollers == 0) { @@ -771,6 +772,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; + gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%"PRId64".%09d deadline=%"PRId64".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, pollset->root_worker); grpc_error *error = GRPC_ERROR_NONE; if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; From 1814374c6958ca4a0a97f7a85e9cdcb7d5d30e82 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 23:14:29 +0000 Subject: [PATCH 029/177] debug --- src/core/lib/iomgr/ev_epollex_linux.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 2afc20daa53..468acd60c7b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -676,14 +676,19 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + grpc_error *error = GRPC_ERROR_NONE; for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { + gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else if (data_ptr == &pollset->pollset_wakeup) { + gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); /* once we start shutting down we stop consuming the wakeup: the fd is level triggered and non-exclusive, which should result in all pollers waking */ @@ -697,6 +702,9 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; + gpr_log(GPR_DEBUG, + "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", pollset, + is_workqueue, cancel, read_ev, write_ev); if (is_workqueue) { append_error(&error, grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), @@ -812,7 +820,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = fd}; + .data.ptr = (void*)(1+(intptr_t)fd)}; if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != 0) { switch (errno) { From cd0354b851390a4db72a02f389d8dfcd7cc7e3fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 16:19:18 -0700 Subject: [PATCH 030/177] Add tracer --- src/core/lib/iomgr/ev_epoll_linux.c | 1 - src/core/lib/iomgr/ev_epollex_linux.c | 39 ++++++++++++++++++++------- src/core/lib/iomgr/ev_posix.c | 5 ++++ src/core/lib/iomgr/ev_posix.h | 2 ++ 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index d41c164d71f..613b435693c 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -66,7 +66,6 @@ #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -static int grpc_polling_trace = 0; /* Disabled by default */ #define GRPC_POLLING_TRACE(fmt, ...) \ if (grpc_polling_trace) { \ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 468acd60c7b..4c35dab0c3e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -519,7 +519,14 @@ static void pollset_global_shutdown(void) { /* p->po.mu must be held before calling this function */ static grpc_error *pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { - gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d root_worker=%p", p, specific_worker, (void*)gpr_tls_get(&g_current_thread_pollset), (void*)gpr_tls_get(&g_current_thread_worker), p->num_pollers, p->root_worker); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, + "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d " + "root_worker=%p", + p, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset), + (void *)gpr_tls_get(&g_current_thread_worker), p->num_pollers, + p->root_worker); + } if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { if (p->num_pollers == 0) { @@ -676,19 +683,25 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); - gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + } grpc_error *error = GRPC_ERROR_NONE; for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + } grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else if (data_ptr == &pollset->pollset_wakeup) { - gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); + } /* once we start shutting down we stop consuming the wakeup: the fd is level triggered and non-exclusive, which should result in all pollers waking */ @@ -702,9 +715,11 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; - gpr_log(GPR_DEBUG, - "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", pollset, - is_workqueue, cancel, read_ev, write_ev); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, + "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", + pollset, is_workqueue, cancel, read_ev, write_ev); + } if (is_workqueue) { append_error(&error, grpc_wakeup_fd_consume_wakeup(&fd->workqueue_wakeup_fd), @@ -780,7 +795,13 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%"PRId64".%09d deadline=%"PRId64".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, pollset->root_worker); + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 + ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", + pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, + deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, + pollset->root_worker); + } grpc_error *error = GRPC_ERROR_NONE; if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; @@ -820,7 +841,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = (void*)(1+(intptr_t)fd)}; + .data.ptr = (void *)(1 + (intptr_t)fd)}; if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != 0) { switch (errno) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 2663152aeca..a8252b01628 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,11 +44,14 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/ev_epoll_linux.h" #include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" +int grpc_polling_trace = 0; /* Disabled by default */ + /** Default poll() function - a pointer so that it can be overridden by some * tests */ grpc_poll_function_type grpc_poll_function = poll; @@ -117,6 +120,8 @@ static void try_engine(const char *engine) { const char *grpc_get_poll_strategy_name() { return g_poll_strategy_name; } void grpc_event_engine_init(void) { + grpc_register_tracer("polling", &grpc_polling_trace); + char *s = gpr_getenv("GRPC_POLL_STRATEGY"); if (s == NULL) { s = gpr_strdup("all"); diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1a9e5c115ae..7ce1e06f20f 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -42,6 +42,8 @@ #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" +extern int grpc_polling_trace; /* Disabled by default */ + typedef struct grpc_fd grpc_fd; typedef struct grpc_event_engine_vtable { From d6255951dc6ca662ddd9cd12c64f783a8218bef2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 23:51:42 +0000 Subject: [PATCH 031/177] Add a benchmark of repeatedly adding an fd to a pollset --- test/cpp/microbenchmarks/bm_pollset.cc | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 3789deebde1..2a0e96ddf1d 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -149,6 +149,33 @@ static void BM_PollEmptyPollset(benchmark::State& state) { } BENCHMARK(BM_PollEmptyPollset); +static void BM_PollAddFd(benchmark::State& state) { + TrackCounters track_counters; + size_t ps_sz = grpc_pollset_size(); + grpc_pollset* ps = static_cast(gpr_zalloc(ps_sz)); + gpr_mu* mu; + grpc_pollset_init(ps, &mu); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_wakeup_fd wakeup_fd; + GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&wakeup_fd))); + grpc_fd *fd = grpc_fd_create(wakeup_fd.read_fd, "xxx"); + while (state.KeepRunning()) { + grpc_pollset_add_fd(&exec_ctx, ps, fd); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_fd_orphan(&exec_ctx, fd, NULL, NULL, "xxx"); + grpc_closure shutdown_ps_closure; + grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, + grpc_schedule_on_exec_ctx); + gpr_mu_lock(mu); + grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); + gpr_mu_unlock(mu); + grpc_exec_ctx_finish(&exec_ctx); + gpr_free(ps); + track_counters.Finish(state); +} +BENCHMARK(BM_PollAddFd); + class Closure : public grpc_closure { public: virtual ~Closure() {} From 1ad9477da37b8aa92459c75ee22c9841075baaf0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 16:15:19 +0000 Subject: [PATCH 032/177] More debug --- src/core/lib/iomgr/ev_epollex_linux.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4c35dab0c3e..35e4e0543f3 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -530,20 +530,38 @@ static grpc_error *pollset_kick(grpc_pollset *p, if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { if (p->num_pollers == 0) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_without_poller", p); + } p->kicked_without_poller = true; return GRPC_ERROR_NONE; } else { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + } return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); } } else { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + } return GRPC_ERROR_NONE; } } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + } return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + } return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); } else { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_via_cv", p); + } gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } @@ -716,9 +734,11 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, - "PS:%p poll got fd: is_wq=%d cancel=%d read=%d write=%d", - pollset, is_workqueue, cancel, read_ev, write_ev); + gpr_log( + GPR_DEBUG, + "PS:%p poll got fd %p(%d/%d): is_wq=%d cancel=%d read=%d write=%d", + pollset, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, is_workqueue, + cancel, read_ev, write_ev); } if (is_workqueue) { append_error(&error, From 8dff7cf0448c727bb9da2687c1b872235599cc10 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 15:40:22 +0000 Subject: [PATCH 033/177] Fix broken test --- test/core/end2end/tests/streaming_error_response.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/end2end/tests/streaming_error_response.c b/test/core/end2end/tests/streaming_error_response.c index c652d9469d5..e032b339880 100644 --- a/test/core/end2end/tests/streaming_error_response.c +++ b/test/core/end2end/tests/streaming_error_response.c @@ -182,6 +182,9 @@ static void test(grpc_end2end_test_config config, bool request_status_early) { GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + if (!request_status_early) { + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + } cq_verify(cqv); memset(ops, 0, sizeof(ops)); @@ -193,9 +196,6 @@ static void test(grpc_end2end_test_config config, bool request_status_early) { GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(103), 1); - if (!request_status_early) { - CQ_EXPECT_COMPLETION(cqv, tag(1), 1); - } cq_verify(cqv); memset(ops, 0, sizeof(ops)); From 67e149a236f7dfad7e97455689893e26ec1b667c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 15:48:39 +0000 Subject: [PATCH 034/177] Fix broken test --- test/core/end2end/tests/keepalive_timeout.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/core/end2end/tests/keepalive_timeout.c b/test/core/end2end/tests/keepalive_timeout.c index bf6ca0d9d94..b53a50d7a1e 100644 --- a/test/core/end2end/tests/keepalive_timeout.c +++ b/test/core/end2end/tests/keepalive_timeout.c @@ -188,8 +188,6 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { GPR_ASSERT(GRPC_CALL_OK == error); CQ_EXPECT_COMPLETION(cqv, tag(102), 1); - cq_verify(cqv); - CQ_EXPECT_COMPLETION(cqv, tag(1), 1); cq_verify(cqv); From 66030c14cdd8710a9220d5fec45f0dd006c10fd6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 16:41:14 +0000 Subject: [PATCH 035/177] Fix broken test --- src/core/lib/iomgr/ev_epollex_linux.c | 5 +++++ test/core/end2end/tests/resource_quota_server.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 35e4e0543f3..5eb92e310ac 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -697,6 +697,11 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, GRPC_SCHEDULING_START_BLOCKING_REGION; int timeout = poll_deadline_to_millis_timeout(deadline, now); + + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p poll for %dms", pollset, timeout); + } + int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c index 4d3ce1c937c..d6d640b8dd7 100644 --- a/test/core/end2end/tests/resource_quota_server.c +++ b/test/core/end2end/tests/resource_quota_server.c @@ -203,7 +203,7 @@ void resource_quota_server(grpc_end2end_test_config config) { op = ops; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 0; - op->flags = 0; + op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY; op->reserved = NULL; op++; op->op = GRPC_OP_SEND_MESSAGE; From 6462fd81b3ae4d2805ab87110238f62996ccb0c5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 17:32:07 +0000 Subject: [PATCH 036/177] Debug --- src/core/lib/iomgr/ev_epollex_linux.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 5eb92e310ac..88b7c83933c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1003,17 +1003,11 @@ static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { } } -#define POTYPES(a, b) (((a)*PO_COUNT) + (b)) - static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { - GPR_ASSERT(a != b); - switch (POTYPES(a->type, b->type)) { - case POTYPES(PO_FD, PO_POLLSET): - pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); - break; - case POTYPES(PO_POLLSET, PO_FD): - pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); - break; + if (a->type == PO_FD && b->type == PO_POLLSET) { + pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); + } else if (a->type == PO_POLLSET && b->type == PO_FD) { + pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); } } @@ -1064,7 +1058,7 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, polling_group *po_group = pg_ref(po->group); gpr_mu_unlock(&po->mu); pg_merge(exec_ctx, pg, po_group); - /* early exit: polling obj picked up a group before joining: we now need + /* early exit: polling obj picked up a group before joining: we needed to do a full merge */ return; } From 361489496541ce51dab11599b441d36852720093 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 17:45:08 +0000 Subject: [PATCH 037/177] Fix mem leaks --- src/core/lib/iomgr/ev_epollex_linux.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 88b7c83933c..db35e0e11f8 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1078,7 +1078,11 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, polling_group *b) { for (;;) { - if (a == b) return; + if (a == b) { + pg_unref(a); + pg_unref(b); + return; + } if (a > b) GPR_SWAP(polling_group *, a, b); gpr_mu_lock(&a->po.mu); gpr_mu_lock(&b->po.mu); @@ -1128,6 +1132,7 @@ static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, pg_unref(unref[i]); } gpr_free(unref); + pg_unref(b); } /******************************************************************************* From 4f98e25f8be7f77e026141adcb40bd079441be40 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 18:55:46 +0000 Subject: [PATCH 038/177] Better cost estimation --- test/cpp/qps/gen_build_yaml.py | 1 + tools/run_tests/generated/tests.json | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 2f035abeddc..805b0faeece 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -66,6 +66,7 @@ def _scenario_json_string(scenario_json, is_tsan): def threads_required(scenario_json, where, is_tsan): scenario_json = mutate_scenario(scenario_json, is_tsan) if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): + if scenario_json['client_config']['client_channels'] == 1: return 1 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) return scenario_json['client_config']['outstanding_rpcs_per_channel'] * scenario_json['client_config']['client_channels'] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a08caf30d36..e58ef3e7d6e 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -41330,7 +41330,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41432,7 +41432,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41484,7 +41484,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41909,7 +41909,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42011,7 +42011,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42063,7 +42063,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42536,7 +42536,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42686,7 +42686,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42762,7 +42762,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43391,7 +43391,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43541,7 +43541,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43617,7 +43617,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", From 3594aa896d2d5a1b0ab56d1d2e6d7998c1cee073 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 14:03:53 -0700 Subject: [PATCH 039/177] Delete unused fields --- src/core/lib/iomgr/ev_epollex_linux.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index db35e0e11f8..a8d7b54887d 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -184,11 +184,6 @@ struct grpc_pollset { grpc_closure *shutdown_closure; grpc_wakeup_fd pollset_wakeup; grpc_pollset_worker *root_worker; - - grpc_pollset *pss_next; - grpc_pollset *pss_prev; - int pss_refs; - grpc_pollset_set *pss_master; }; /******************************************************************************* From 7a59b56e53020e655f83d65c165dfb8c228ddf4d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 14:20:47 -0700 Subject: [PATCH 040/177] sketching pollable set --- src/core/lib/iomgr/ev_epollex_linux.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index a8d7b54887d..0b3e2b13fc8 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -177,7 +177,11 @@ struct grpc_pollset_worker { struct grpc_pollset { polling_obj po; - int epfd; + /* Pollable set - possible values: + 0 - nothing is pollable + pointer | 1 - a single pollable file descriptor + (fd << 1) | 0 - an epoll fd */ + gpr_atm pollable_set_atm; int num_pollers; bool kicked_without_poller; gpr_atm shutdown_atm; From 9f012514738c560df9ccd72971390a5b097256c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 15:37:14 -0700 Subject: [PATCH 041/177] Working towards a single-fd optimized data structure --- src/core/lib/iomgr/ev_epollex_linux.c | 161 +++++++++++++++++--------- 1 file changed, 109 insertions(+), 52 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0b3e2b13fc8..6e6e4460c02 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -170,24 +170,27 @@ static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { struct grpc_pollset_worker { bool kicked; bool initialized_cv; + bool inserted; gpr_cv cv; grpc_pollset_worker *next; grpc_pollset_worker *prev; + grpc_pollset *pollset; }; -struct grpc_pollset { +struct pollable { polling_obj po; - /* Pollable set - possible values: - 0 - nothing is pollable - pointer | 1 - a single pollable file descriptor - (fd << 1) | 0 - an epoll fd */ - gpr_atm pollable_set_atm; + int epfd; + grpc_wakeup_fd wakeup; + grpc_pollset_worker *root_worker; int num_pollers; - bool kicked_without_poller; gpr_atm shutdown_atm; +}; + +struct grpc_pollset { + pollable pollable; + pollable *current_pollable; + bool kicked_without_poller; grpc_closure *shutdown_closure; - grpc_wakeup_fd pollset_wakeup; - grpc_pollset_worker *root_worker; }; /******************************************************************************* @@ -573,32 +576,53 @@ static grpc_error *kick_poller(void) { static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { po_init(&pollset->po, PO_POLLSET); pollset->kicked_without_poller = false; - pollset->epfd = epoll_create1(EPOLL_CLOEXEC); - if (pollset->epfd < 0) { - GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_create1")); + gpr_atm_no_barrier_store(&pollset->pollable_set_atm, 0); + pollset->num_pollers = 0; + gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); + pollset->shutdown_closure = NULL; + pollset->root_worker = NULL; + *mu = &pollset->po.mu; +} + +static grpc_error *multipoller_create(multipoller **out) { + multipoller *p = gpr_malloc(sizeof(*p)); + p->epfd = epoll_create1(EPOLL_CLOEXEC); + if (p->epfd < 0) { + grpc_error *err = GRPC_OS_ERROR(errno, "epoll_create1"); + gpr_free(p); + return err; } else { struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, .data.ptr = &global_wakeup_fd}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, - &ev) != 0) { - GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); + if (epoll_ctl(p->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { + grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(p->epfd); + gpr_free(p); + return err; } } - pollset->num_pollers = 0; - gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); - pollset->shutdown_closure = NULL; - if (GRPC_LOG_IF_ERROR("pollset_init", - grpc_wakeup_fd_init(&pollset->pollset_wakeup)) && - pollset->epfd >= 0) { - struct epoll_event ev = {.events = EPOLLIN | EPOLLET, - .data.ptr = &pollset->pollset_wakeup}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, pollset->pollset_wakeup.read_fd, - &ev) != 0) { - GRPC_LOG_IF_ERROR("pollset_init", GRPC_OS_ERROR(errno, "epoll_ctl")); - } + grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); + if (err != GRPC_ERROR_NONE) { + close(p->epfd); + gpr_free(p); + return err; } - pollset->root_worker = NULL; - *mu = &pollset->po.mu; + struct epoll_event ev = {.events = EPOLLIN | EPOLLET, .data.ptr = &p->wakeup}; + if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { + err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + gpr_free(p); + return err; + } + *out = p; + return GRPC_ERROR_NONE; +} + +static void multipoller_destroy(multipoller *p) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + gpr_free(p); } /* Convert a timespec to milliseconds: @@ -678,31 +702,40 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { po_destroy(&pollset->po); - if (pollset->epfd >= 0) close(pollset->epfd); - grpc_wakeup_fd_destroy(&pollset->pollset_wakeup); + switch (pollset->occupancy) { + case POLLSET_EMPTY: + break; + case POLLSET_UNARY_FD: + UNREF_BY(pollset->pollable.unary_fd, 2); + break; + case POLLSET_MULTIPOLLER: + multipoller_destroy(pollset->pollable.multipoller); + break; + } } #define MAX_EPOLL_EVENTS 100 -static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - gpr_timespec now, gpr_timespec deadline) { +static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { struct epoll_event events[MAX_EPOLL_EVENTS]; static const char *err_desc = "pollset_poll"; - if (pollset->epfd < 0) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "epoll fd failed to initialize"); - } - - GRPC_SCHEDULING_START_BLOCKING_REGION; int timeout = poll_deadline_to_millis_timeout(deadline, now); if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p poll for %dms", pollset, timeout); } - int r = epoll_wait(pollset->epfd, events, MAX_EPOLL_EVENTS, timeout); - GRPC_SCHEDULING_END_BLOCKING_REGION; + if (timeout != 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; + } + int r = epoll_wait(pollset->pollable.multipoller->epfd, events, + MAX_EPOLL_EVENTS, timeout); + if (timeout != 0) { + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); if (grpc_polling_trace) { @@ -720,7 +753,7 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); - } else if (data_ptr == &pollset->pollset_wakeup) { + } else if (data_ptr == &pollset->pollable.multipoller->pollset_wakeup) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); } @@ -728,7 +761,9 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, the fd is level triggered and non-exclusive, which should result in all pollers waking */ if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { - append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + append_error(&error, + grpc_wakeup_fd_consume_wakeup( + &pollset->pollable.multipoller->pollset_wakeup), err_desc); } } else { @@ -767,20 +802,29 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec deadline) { - if (worker_hdl != NULL) { - *worker_hdl = worker; + worker->initialized_cv = false; + worker->inserted = false; + if (worker_hdl != NULL || pollset->occupancy != POLLSET_MULTIPOLLER) { + if (worker_hdl != NULL) *worker_hdl = worker; worker->kicked = false; + worker->inserted = true; if (pollset->root_worker == NULL) { pollset->root_worker = worker; worker->next = worker->prev = worker; - worker->initialized_cv = false; + if (pollset->occupancy == POLLSET_EMPTY) { + worker->initialized_cv = true; + } } else { worker->next = pollset->root_worker; worker->prev = worker->next->prev; worker->next->prev = worker->prev->next = worker; worker->initialized_cv = true; + } + if (worker->initialized_cv) { + GPR_ASSERT(worker->inserted); gpr_cv_init(&worker->cv); - while (pollset->root_worker != worker) { + while (pollset->root_worker != worker || + pollset->occupancy == POLLSET_EMPTY) { if (gpr_cv_wait(&worker->cv, &pollset->po.mu, deadline)) return false; if (worker->kicked) return false; } @@ -791,7 +835,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (worker_hdl != NULL) { + if (worker->inserted) { if (worker == pollset->root_worker) { if (worker == worker->next) { pollset->root_worker = NULL; @@ -819,6 +863,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; + grpc_pollset_worker *fake_worker_hdl; if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", @@ -836,10 +881,22 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); pollset->num_pollers++; - gpr_mu_unlock(&pollset->po.mu); - error = pollset_poll(exec_ctx, pollset, now, deadline); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); + switch (pollset->occupancy) { + case POLLSET_EMPTY: + GPR_UNREACHABLE_CODE(break); + case POLLSET_UNARY_FD: + gpr_mu_unlock(&pollset->po.mu); + error = pollset_poll(exec_ctx, pollset, now, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + break; + case POLLSET_MULTIPOLLER: + gpr_mu_unlock(&pollset->po.mu); + error = pollset_epoll(exec_ctx, pollset, now, deadline); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + break; + } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset->num_pollers--; From d1d7fdd791e2ad62427a1bbe7d918d8fd822d8d4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 16:16:24 -0700 Subject: [PATCH 042/177] Sketch turnstile with single-fd-poller optimization --- src/core/lib/iomgr/ev_epollex_linux.c | 594 ++++++++++++++++---------- 1 file changed, 358 insertions(+), 236 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 6e6e4460c02..d49654bbc26 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -77,10 +77,11 @@ static grpc_wakeup_fd global_wakeup_fd; */ typedef enum { - PO_FD, - PO_POLLSET, - PO_POLLSET_SET, PO_POLLING_GROUP, + PO_POLLSET_SET, + PO_POLLSET, + PO_FD, /* ordering is important: we always want to lock pollsets before fds: + this guarantees that using an fd as a pollable is safe */ PO_COUNT } polling_obj_type; @@ -103,6 +104,7 @@ struct polling_group { static void po_init(polling_obj *po, polling_obj_type type); static void po_destroy(polling_obj *po); static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b); +static int po_cmp(polling_obj *a, polling_obj *b); static void pg_create(grpc_exec_ctx *exec_ctx, polling_obj **initial_po, size_t initial_po_count); @@ -113,12 +115,30 @@ static void pg_merge(grpc_exec_ctx *exec_ctx, polling_group *a, static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, polling_obj *po); +/******************************************************************************* + * pollable Declarations + */ + +typedef struct pollable { + polling_obj po; + int epfd; + grpc_wakeup_fd wakeup; + grpc_pollset_worker *root_worker; +} pollable; + +static pollable g_empty_pollable; + +static void pollable_init(pollable *p, polling_obj_type type); +static void pollable_destroy(pollable *p); +/* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ +static grpc_error *pollable_materialize(pollable *p); + /******************************************************************************* * Fd Declarations */ struct grpc_fd { - polling_obj po; + pollable pollable; int fd; /* refst format: bit 0 : 1=Active / 0=Orphaned @@ -167,23 +187,25 @@ static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { /******************************************************************************* * Pollset Declarations */ + +typedef struct pollset_worker_link { + grpc_pollset_worker *next; + grpc_pollset_worker *prev; +} pollset_worker_link; + +typedef enum { + PWL_POLLSET, + PWL_POLLABLE, + POLLSET_WORKER_LINK_COUNT +} pollset_worker_links; + struct grpc_pollset_worker { bool kicked; bool initialized_cv; - bool inserted; + pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; gpr_cv cv; - grpc_pollset_worker *next; - grpc_pollset_worker *prev; grpc_pollset *pollset; -}; - -struct pollable { - polling_obj po; - int epfd; - grpc_wakeup_fd wakeup; - grpc_pollset_worker *root_worker; - int num_pollers; - gpr_atm shutdown_atm; + pollable *pollable; }; struct grpc_pollset { @@ -191,6 +213,7 @@ struct grpc_pollset { pollable *current_pollable; bool kicked_without_poller; grpc_closure *shutdown_closure; + grpc_pollset_worker *root_worker; }; /******************************************************************************* @@ -270,7 +293,7 @@ static void unref_by(grpc_fd *fd, int n) { if (old == n) { /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); - po_destroy(&fd->po); + pollable_destroy(&fd->pollable); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; @@ -311,7 +334,7 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = gpr_malloc(sizeof(grpc_fd)); } - po_init(&new_fd->po, PO_FD); + pollable_init(&new_fd->pollable, PO_FD); gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; @@ -342,11 +365,11 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { int ret_fd = -1; - gpr_mu_lock(&fd->po.mu); + gpr_mu_lock(&fd->pollable.po.mu); if (!fd->orphaned) { ret_fd = fd->fd; } - gpr_mu_unlock(&fd->po.mu); + gpr_mu_unlock(&fd->pollable.po.mu); return ret_fd; } @@ -357,7 +380,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - gpr_mu_lock(&fd->po.mu); + gpr_mu_lock(&fd->pollable.po.mu); fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file @@ -381,7 +404,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); - gpr_mu_unlock(&fd->po.mu); + gpr_mu_unlock(&fd->pollable.po.mu); UNREF_BY(fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -499,9 +522,94 @@ static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { return &((grpc_fd *)workqueue)->workqueue_scheduler; } +/******************************************************************************* + * Pollable Definitions + */ + +static void pollable_init(pollable *p, polling_obj_type type) { + po_init(&p->po, type); + p->root_worker = NULL; + p->epfd = -1; +} + +static void pollable_destroy(pollable *p) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + po_destroy(&p->po); +} + +/* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ +static grpc_error *pollable_materialize(pollable *p) { + if (p->epfd == -1) { + int new_epfd = epoll_create1(EPOLL_CLOEXEC); + if (new_epfd < 0) { + return GRPC_OS_ERROR(errno, "epoll_create1"); + } else { + struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != + 0) { + grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(new_epfd); + return err; + } + } + grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); + if (err != GRPC_ERROR_NONE) { + close(new_epfd); + return err; + } + struct epoll_event ev = {.events = EPOLLIN | EPOLLET, + .data.ptr = &p->wakeup}; + if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { + err = GRPC_OS_ERROR(errno, "epoll_ctl"); + close(new_epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + return err; + } + + p->epfd = new_epfd; + } + return GRPC_ERROR_NONE; +} + +/* pollable must be materialized */ +static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollable_add_fd"; + const int epfd = p->epfd; + + struct epoll_event ev_fd = { + .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { + switch (errno) { + case EEXIST: /* if this fd is already in the epoll set, the workqueue fd + must also be - just return */ + return GRPC_ERROR_NONE; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, + .data.ptr = (void *)(1 + (intptr_t)fd)}; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != + 0) { + switch (errno) { + case EEXIST: /* if the workqueue fd is already in the epoll set we're ok + - no need to do anything special */ + break; + default: + append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + } + } + + return error; +} + /******************************************************************************* * Pollset Definitions */ + GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); @@ -518,51 +626,78 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } +static grpc_error *pollset_kick_all(grpc_pollset *pollset) { + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->root_worker != NULL) { + grpc_pollset_worker *worker = pollset->root_worker; + do { + if (worker->initialized_cv) { + gpr_cv_signal(&worker->cv); + } else { + append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), + "pollset_shutdown"); + } + + worker = worker->links[PWL_POLLSET].next; + } while (worker != pollset->root_worker); + } + return error; +} + /* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *p, +static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable) { + gpr_mu_lock(&p->po.mu); + } + if (grpc_polling_trace) { gpr_log(GPR_DEBUG, - "PS:%p kick %p tls_pollset=%p tls_worker=%p num_pollers=%d " - "root_worker=%p", + "PS:%p kick %p tls_pollset=%p tls_worker=%p " + "root_worker=(pollset:%p pollable:%p)", p, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset), - (void *)gpr_tls_get(&g_current_thread_worker), p->num_pollers, + (void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker, p->root_worker); } if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { - if (p->num_pollers == 0) { + if (pollset->root_worker == NULL) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_without_poller", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); } - p->kicked_without_poller = true; + pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; } else { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_any_via_wakeup_fd", p); } - return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + grpc_error *err = pollable_materialize(p); + if (err != GRPC_ERROR_NONE) return err; + return grpc_wakeup_fd_wakeup(&p->wakeup); } } else { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_any_but_awake", p); } return GRPC_ERROR_NONE; } } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_but_awake", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); } return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_via_wakeup_fd", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_wakeup_fd", p); } - return grpc_wakeup_fd_wakeup(&p->pollset_wakeup); + grpc_error *err = pollable_materialize(p); + if (err != GRPC_ERROR_NONE) return err; + return grpc_wakeup_fd_wakeup(&p->wakeup); } else { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p kicked_via_cv", p); + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); } gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; @@ -574,55 +709,12 @@ static grpc_error *kick_poller(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - po_init(&pollset->po, PO_POLLSET); + pollable_init(&pollset->pollable, PO_POLLSET); + pollset->current_pollable = &g_empty_pollable; pollset->kicked_without_poller = false; - gpr_atm_no_barrier_store(&pollset->pollable_set_atm, 0); - pollset->num_pollers = 0; - gpr_atm_no_barrier_store(&pollset->shutdown_atm, 0); pollset->shutdown_closure = NULL; pollset->root_worker = NULL; - *mu = &pollset->po.mu; -} - -static grpc_error *multipoller_create(multipoller **out) { - multipoller *p = gpr_malloc(sizeof(*p)); - p->epfd = epoll_create1(EPOLL_CLOEXEC); - if (p->epfd < 0) { - grpc_error *err = GRPC_OS_ERROR(errno, "epoll_create1"); - gpr_free(p); - return err; - } else { - struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, - .data.ptr = &global_wakeup_fd}; - if (epoll_ctl(p->epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { - grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); - close(p->epfd); - gpr_free(p); - return err; - } - } - grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); - if (err != GRPC_ERROR_NONE) { - close(p->epfd); - gpr_free(p); - return err; - } - struct epoll_event ev = {.events = EPOLLIN | EPOLLET, .data.ptr = &p->wakeup}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { - err = GRPC_OS_ERROR(errno, "epoll_ctl"); - close(p->epfd); - grpc_wakeup_fd_destroy(&p->wakeup); - gpr_free(p); - return err; - } - *out = p; - return GRPC_ERROR_NONE; -} - -static void multipoller_destroy(multipoller *p) { - close(p->epfd); - grpc_wakeup_fd_destroy(&p->wakeup); - gpr_free(p); + *mu = &pollset->pollable.po.mu; } /* Convert a timespec to milliseconds: @@ -667,9 +759,20 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } +static grpc_error *fd_become_pollable(grpc_fd *fd) { + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "fd_become_pollable"; + gpr_mu_lock(&fd->pollable.po.mu); + if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { + append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); + } + gpr_mu_unlock(&fd->pollable.po.mu); + return error; +} + static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - if (pollset->shutdown_closure != NULL && pollset->num_pollers == 0) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); } } @@ -679,45 +782,27 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { GPR_ASSERT(pollset->shutdown_closure == NULL); pollset->shutdown_closure = closure; - gpr_atm_no_barrier_store(&pollset->shutdown_atm, 1); - if (pollset->num_pollers > 0) { - struct epoll_event ev = {.events = EPOLLIN, - .data.ptr = &pollset->pollset_wakeup}; - epoll_ctl(pollset->epfd, EPOLL_CTL_MOD, pollset->pollset_wakeup.read_fd, - &ev); - GRPC_LOG_IF_ERROR("pollset_shutdown", - grpc_wakeup_fd_wakeup(&pollset->pollset_wakeup)); - } - if (pollset->root_worker != NULL) { - for (grpc_pollset_worker *worker = pollset->root_worker->next; - worker != pollset->root_worker; worker = worker->next) { - if (worker->initialized_cv) { - gpr_cv_signal(&worker->cv); - } - } - } + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); pollset_maybe_finish_shutdown(exec_ctx, pollset); } +static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { + return p != &g_empty_pollable && p != &pollset->pollable; +} + /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ static void pollset_destroy(grpc_pollset *pollset) { - po_destroy(&pollset->po); - switch (pollset->occupancy) { - case POLLSET_EMPTY: - break; - case POLLSET_UNARY_FD: - UNREF_BY(pollset->pollable.unary_fd, 2); - break; - case POLLSET_MULTIPOLLER: - multipoller_destroy(pollset->pollable.multipoller); - break; + pollable_destroy(&pollset->pollable); + if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { + UNREF_BY((grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); } } #define MAX_EPOLL_EVENTS 100 static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - gpr_timespec now, gpr_timespec deadline) { + pollable *p, gpr_timespec now, + gpr_timespec deadline) { struct epoll_event events[MAX_EPOLL_EVENTS]; static const char *err_desc = "pollset_poll"; @@ -730,8 +815,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (timeout != 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; } - int r = epoll_wait(pollset->pollable.multipoller->epfd, events, - MAX_EPOLL_EVENTS, timeout); + int r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); if (timeout != 0) { GRPC_SCHEDULING_END_BLOCKING_REGION; } @@ -753,19 +837,11 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_timer_consume_kick(); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); - } else if (data_ptr == &pollset->pollable.multipoller->pollset_wakeup) { + } else if (data_ptr == &p->wakeup) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); } - /* once we start shutting down we stop consuming the wakeup: - the fd is level triggered and non-exclusive, which should result in all - pollers waking */ - if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { - append_error(&error, - grpc_wakeup_fd_consume_wakeup( - &pollset->pollable.multipoller->pollset_wakeup), - err_desc); - } + append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); } else { grpc_fd *fd = (grpc_fd *)(((intptr_t)data_ptr) & ~(intptr_t)1); bool is_workqueue = (((intptr_t)data_ptr) & 1) != 0; @@ -798,60 +874,89 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } +/* Return true if first in list */ +static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, + grpc_pollset_worker *worker) { + if (*root == NULL) { + *root = worker; + worker->links[link].next = worker->links[link].prev = worker; + return true; + } else { + worker->links[link].next = *root; + worker->links[link].prev = worker->links[link].next->links[link].prev; + worker->links[link].next->links[link].prev = worker; + worker->links[link].prev->links[link].next = worker; + return false; + } +} + +/* Return true if last in list */ +typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; + +static worker_remove_result worker_remove(grpc_pollset_worker **root, + pollset_worker_links link, + grpc_pollset_worker *worker) { + if (worker == *root) { + if (worker == worker->links[link].next) { + *root = NULL; + return EMPTIED; + } else { + *root = worker->links[link].next; + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return NEW_ROOT; + } + } else { + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return REMOVED; + } +} + /* Return true if this thread should poll */ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec deadline) { + if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; - worker->inserted = false; - if (worker_hdl != NULL || pollset->occupancy != POLLSET_MULTIPOLLER) { - if (worker_hdl != NULL) *worker_hdl = worker; - worker->kicked = false; - worker->inserted = true; - if (pollset->root_worker == NULL) { - pollset->root_worker = worker; - worker->next = worker->prev = worker; - if (pollset->occupancy == POLLSET_EMPTY) { - worker->initialized_cv = true; + worker->kicked = false; + worker->pollset = pollset; + worker->pollable = pollset->current_pollable; + + if (pollset_is_pollable_fd(pollset, worker->pollable)) { + REF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + } + + worker_insert(&pollset->root_worker, PWL_POLLSET, worker); + if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (pollset->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &pollset->current_pollable->po.mu, + deadline)) { + return false; } - } else { - worker->next = pollset->root_worker; - worker->prev = worker->next->prev; - worker->next->prev = worker->prev->next = worker; - worker->initialized_cv = true; - } - if (worker->initialized_cv) { - GPR_ASSERT(worker->inserted); - gpr_cv_init(&worker->cv); - while (pollset->root_worker != worker || - pollset->occupancy == POLLSET_EMPTY) { - if (gpr_cv_wait(&worker->cv, &pollset->po.mu, deadline)) return false; - if (worker->kicked) return false; + if (worker->kicked) { + return false; } } } + return pollset->shutdown_closure == NULL; } static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (worker->inserted) { - if (worker == pollset->root_worker) { - if (worker == worker->next) { - pollset->root_worker = NULL; - } else { - pollset->root_worker = worker->next; - worker->prev->next = worker->next; - worker->next->prev = worker->prev; - gpr_cv_signal(&pollset->root_worker->cv); - } - } else { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; - } - if (worker->initialized_cv) { - gpr_cv_destroy(&worker->cv); - } + worker_remove(&pollset->root_worker, PWL_POLLSET, worker); + if (NEW_ROOT == + worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&worker->pollable->root_worker->cv); + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + if (pollset_is_pollable_fd(pollset, worker->pollable)) { + UNREF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); } } @@ -863,7 +968,6 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - grpc_pollset_worker *fake_worker_hdl; if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", @@ -872,67 +976,71 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->root_worker); } grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_work"; if (pollset->kicked_without_poller) { pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } + if (pollset->current_pollable != &pollset->pollable) { + gpr_mu_lock(&pollset->current_pollable->po.mu); + } if (begin_worker(pollset, &worker, worker_hdl, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - pollset->num_pollers++; - switch (pollset->occupancy) { - case POLLSET_EMPTY: - GPR_UNREACHABLE_CODE(break); - case POLLSET_UNARY_FD: - gpr_mu_unlock(&pollset->po.mu); - error = pollset_poll(exec_ctx, pollset, now, deadline); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); - break; - case POLLSET_MULTIPOLLER: - gpr_mu_unlock(&pollset->po.mu); - error = pollset_epoll(exec_ctx, pollset, now, deadline); - grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->po.mu); - break; + append_error(&error, pollable_materialize(worker.pollable), err_desc); + if (worker.pollable != &pollset->pollable) { + gpr_mu_unlock(&worker.pollable->po.mu); + } + gpr_mu_unlock(&pollset->pollable.po.mu); + append_error(&error, pollset_epoll(exec_ctx, pollset, worker.pollable, now, + deadline), + err_desc); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->pollable.po.mu); + if (worker.pollable != &pollset->pollable) { + gpr_mu_lock(&worker.pollable->po.mu); } gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); - pollset->num_pollers--; pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(pollset, &worker, worker_hdl); + if (worker.pollable != &pollset->pollable) { + gpr_mu_unlock(&worker.pollable->po.mu); + } return error; } static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; + grpc_fd *unref_fd = NULL; static const char *err_desc = "pollset_add_fd"; - struct epoll_event ev_fd = { - .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { - switch (errno) { - case EEXIST: /* if this fd is already in the epoll set, the workqueue fd - must also be - just return */ - return; - default: - append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); + gpr_mu_lock(&pollset->pollable.po.mu); + if (pollset->current_pollable == &g_empty_pollable) { + /* empty pollable --> single fd pollable */ + append_error(&error, pollset_kick_all(pollset), err_desc); + pollset->current_pollable = &fd->pollable; + append_error(&error, fd_become_pollable(fd), err_desc); + REF_BY(fd, 2, "pollset_pollable"); + } else if (pollset->current_pollable == &pollset->pollable) { + append_error(&error, pollable_add_fd(pollset->current_pollable, fd), + err_desc); + } else if (pollset->current_pollable != &fd->pollable) { + unref_fd = (grpc_fd *)pollset->current_pollable; + pollset->current_pollable = &pollset->pollable; + if (append_error(&error, pollable_materialize(&pollset->pollable), + err_desc)) { + pollable_add_fd(&pollset->pollable, unref_fd); + pollable_add_fd(&pollset->pollable, fd); } } - struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = (void *)(1 + (intptr_t)fd)}; - if (epoll_ctl(pollset->epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, - &ev_wq) != 0) { - switch (errno) { - case EEXIST: /* if the workqueue fd is already in the epoll set we're ok - - no need to do anything special */ - break; - default: - append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); - } + gpr_mu_unlock(&pollset->pollable.po.mu); + if (unref_fd) { + UNREF_BY(unref_fd, 2, "pollset_pollable"); } + GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -954,7 +1062,7 @@ static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_fd *fd) { - po_join(exec_ctx, &pss->po, &fd->po); + po_join(exec_ctx, &pss->po, &fd->pollable.po); } static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, @@ -962,7 +1070,7 @@ static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, grpc_pollset *ps) { - po_join(exec_ctx, &pss->po, &ps->po); + po_join(exec_ctx, &pss->po, &ps->pollable.po); } static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, @@ -1022,40 +1130,54 @@ static void pg_unref(polling_group *pg) { } } +static int po_cmp(polling_obj *a, polling_obj *b) { + if (a == b) return 0; + if (a->type < b->type) return -1; + if (a->type > b->type) return 1; + if (a < b) return -1; + assert(a > b); + return 1; +} + static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { - if (a == b) return; - if (a > b) GPR_SWAP(polling_obj *, a, b); - - gpr_mu_lock(&a->mu); - gpr_mu_lock(&b->mu); - - if (a->group == NULL) { - if (b->group == NULL) { - polling_obj *initial_po[] = {a, b}; - pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - } else { - polling_group *b_group = pg_ref(b->group); - gpr_mu_unlock(&b->mu); - gpr_mu_unlock(&a->mu); - pg_join(exec_ctx, b_group, a); - } - } else if (b->group == NULL) { - polling_group *a_group = pg_ref(a->group); - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - pg_join(exec_ctx, a_group, b); - } else if (a->group == b->group) { - /* nothing to do */ - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - } else { - polling_group *a_group = pg_ref(a->group); - polling_group *b_group = pg_ref(b->group); - gpr_mu_unlock(&a->mu); - gpr_mu_unlock(&b->mu); - pg_merge(exec_ctx, a_group, b_group); + switch (po_cmp(a, b)) { + case 0: + return; + case 1: + GPR_SWAP(polling_obj *, a, b); + /* fall through */ + case -1: + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + + if (a->group == NULL) { + if (b->group == NULL) { + polling_obj *initial_po[] = {a, b}; + pg_create(exec_ctx, initial_po, GPR_ARRAY_SIZE(initial_po)); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + pg_join(exec_ctx, b_group, a); + } + } else if (b->group == NULL) { + polling_group *a_group = pg_ref(a->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_join(exec_ctx, a_group, b); + } else if (a->group == b->group) { + /* nothing to do */ + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + } else { + polling_group *a_group = pg_ref(a->group); + polling_group *b_group = pg_ref(b->group); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); + pg_merge(exec_ctx, a_group, b_group); + } } } From 911490c0d4edf4804304121cf63f81a6336b66cf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 16:19:57 -0700 Subject: [PATCH 043/177] Finish initialization code --- src/core/lib/iomgr/ev_epollex_linux.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d49654bbc26..d01aeb9c821 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -82,6 +82,7 @@ typedef enum { PO_POLLSET, PO_FD, /* ordering is important: we always want to lock pollsets before fds: this guarantees that using an fd as a pollable is safe */ + PO_EMPTY_POLLABLE, PO_COUNT } polling_obj_type; @@ -612,16 +613,23 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); +static bool global_wakeup_fd_initialized = false; /* Global state management */ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_global_init"; + global_wakeup_fd_initialized = + append_error(&error, grpc_wakeup_fd_init(&global_wakeup_fd), err_desc); + pollable_init(&g_empty_pollable, PO_EMPTY_POLLABLE); + return error; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); + if (global_wakeup_fd_initialized) grpc_wakeup_fd_destroy(&global_wakeup_fd); + pollable_destroy(&g_empty_pollable); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } From 54d388d0af69aae8ba7716da7f6c3bc8b1768664 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 16:20:45 -0700 Subject: [PATCH 044/177] init tweaks --- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d01aeb9c821..0918b22e462 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1438,6 +1438,8 @@ const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + pollset_global_shutdown(); + fd_global_shutdown(); return NULL; } From 29a256773a23a530814d9b28d26ab99674de59c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 15 Apr 2017 00:00:39 +0000 Subject: [PATCH 045/177] Consume the orrect fd --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index db35e0e11f8..4ec9774b0c1 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -729,7 +729,7 @@ static grpc_error *pollset_poll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, the fd is level triggered and non-exclusive, which should result in all pollers waking */ if (gpr_atm_no_barrier_load(&pollset->shutdown_atm) == 0) { - append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + append_error(&error, grpc_wakeup_fd_consume_wakeup(&pollset->pollset_wakeup), err_desc); } } else { From 26017de5d55f3ec5002afa0b97c7fe660d6144bf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 15 Apr 2017 00:07:25 +0000 Subject: [PATCH 046/177] Fix leaked lock --- src/core/lib/iomgr/ev_epollex_linux.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0918b22e462..2c0fe42acc8 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -652,14 +652,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { return error; } -/* p->po.mu must be held before calling this function */ -static grpc_error *pollset_kick(grpc_pollset *pollset, - grpc_pollset_worker *specific_worker) { - pollable *p = pollset->current_pollable; - if (p != &pollset->pollable) { - gpr_mu_lock(&p->po.mu); - } - +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p " @@ -710,6 +703,21 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } + +} + +/* p->po.mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { + pollable *p = pollset->current_pollable; + if (p != &pollset->pollable) { + gpr_mu_lock(&p->po.mu); + } + grpc_error *error = pollset_kick_inner(pollset, p, specific_worker); + if (p != &pollset->pollable) { + gpr_mu_unlock(&p->po.mu); + } + return error; } static grpc_error *kick_poller(void) { From e310f5b6230ab38b0c2e9aafb4caf1aa1fc45faa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 15:43:58 +0000 Subject: [PATCH 047/177] Fix scheduled bit --- src/core/lib/iomgr/ev_epollex_linux.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 2c0fe42acc8..eba1bf920c6 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -509,6 +509,9 @@ static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); } else if (gpr_atm_no_barrier_load(&fd->workqueue_item_count) > 0) { From 50480b2a058fc74c3cf934ae5ae9981b2417005e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 16:34:52 +0000 Subject: [PATCH 048/177] Fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 130 +++++++++++++++++--------- src/core/lib/iomgr/pollset.h | 2 +- 2 files changed, 86 insertions(+), 46 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index eba1bf920c6..fdde21756bf 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -265,7 +265,7 @@ static gpr_mu fd_freelist_mu; #ifdef GRPC_FD_REF_COUNT_DEBUG #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n, reason, __FILE__, __LINE__) static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, int line) { gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, @@ -273,25 +273,14 @@ static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); #else #define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) +#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n) static void ref_by(grpc_fd *fd, int n) { #endif GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); } -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { +static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { +grpc_fd *fd = arg; /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); pollable_destroy(&fd->pollable); @@ -303,6 +292,22 @@ static void unref_by(grpc_fd *fd, int n) { grpc_lfev_destroy(&fd->write_closure); gpr_mu_unlock(&fd_freelist_mu); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); } else { GPR_ASSERT(old > n); } @@ -406,7 +411,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->pollable.po.mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ + UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); } @@ -459,7 +464,7 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, const char *file, int line, const char *reason) { if (workqueue != NULL) { - unref_by((grpc_fd *)workqueue, 2, file, line, reason); + unref_by(exec_ctx, (grpc_fd *)workqueue, 2, file, line, reason); } } #else @@ -473,7 +478,7 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) { if (workqueue != NULL) { - unref_by((grpc_fd *)workqueue, 2); + unref_by(exec_ctx, (grpc_fd *)workqueue, 2); } } #endif @@ -495,7 +500,7 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, if (last == 0) { workqueue_wakeup(fd); } - UNREF_BY(fd, 2, "workqueue_enqueue"); + UNREF_BY(exec_ctx, fd, 2, "workqueue_enqueue"); } static void fd_invoke_workqueue(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { @@ -778,14 +783,12 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static grpc_error *fd_become_pollable(grpc_fd *fd) { +static grpc_error *fd_become_pollable_locked(grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "fd_become_pollable"; - gpr_mu_lock(&fd->pollable.po.mu); if (append_error(&error, pollable_materialize(&fd->pollable), err_desc)) { append_error(&error, pollable_add_fd(&fd->pollable, fd), err_desc); } - gpr_mu_unlock(&fd->pollable.po.mu); return error; } @@ -810,10 +813,10 @@ static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { } /* pollset_shutdown is guaranteed to be called before pollset_destroy. */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { pollable_destroy(&pollset->pollable); if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY((grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); } } @@ -975,7 +978,7 @@ static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_cv_destroy(&worker->cv); } if (pollset_is_pollable_fd(pollset, worker->pollable)) { - UNREF_BY((grpc_fd *)worker->pollable, 2, "one_poll"); + UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); } } @@ -1031,35 +1034,45 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { - grpc_error *error = GRPC_ERROR_NONE; - grpc_fd *unref_fd = NULL; +static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + grpc_fd *fd = arg; + UNREF_BY(exec_ctx, fd, 2, "pollset_pollable"); +} + +/* expects pollsets locked, flag whether fd is locked or not */ +static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd, bool fd_locked) { static const char *err_desc = "pollset_add_fd"; - gpr_mu_lock(&pollset->pollable.po.mu); + grpc_error *error = GRPC_ERROR_NONE; if (pollset->current_pollable == &g_empty_pollable) { /* empty pollable --> single fd pollable */ append_error(&error, pollset_kick_all(pollset), err_desc); pollset->current_pollable = &fd->pollable; - append_error(&error, fd_become_pollable(fd), err_desc); + if (!fd_locked) gpr_mu_lock(&fd->pollable.po.mu); + append_error(&error, fd_become_pollable_locked(fd), err_desc); + if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); REF_BY(fd, 2, "pollset_pollable"); } else if (pollset->current_pollable == &pollset->pollable) { append_error(&error, pollable_add_fd(pollset->current_pollable, fd), err_desc); } else if (pollset->current_pollable != &fd->pollable) { - unref_fd = (grpc_fd *)pollset->current_pollable; + grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; pollset->current_pollable = &pollset->pollable; if (append_error(&error, pollable_materialize(&pollset->pollable), err_desc)) { - pollable_add_fd(&pollset->pollable, unref_fd); + pollable_add_fd(&pollset->pollable, had_fd); pollable_add_fd(&pollset->pollable, fd); } + grpc_closure_sched(exec_ctx, grpc_closure_create(unref_fd_no_longer_poller, had_fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); } - gpr_mu_unlock(&pollset->pollable.po.mu); - if (unref_fd) { - UNREF_BY(unref_fd, 2, "pollset_pollable"); - } + return error; +} +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + gpr_mu_lock(&pollset->pollable.po.mu); + grpc_error *error = pollset_add_fd_locked(exec_ctx, pollset, fd, false); + gpr_mu_unlock(&pollset->pollable.po.mu); GRPC_LOG_IF_ERROR("pollset_add_fd", error); } @@ -1202,9 +1215,9 @@ static void po_join(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { static void pg_notify(grpc_exec_ctx *exec_ctx, polling_obj *a, polling_obj *b) { if (a->type == PO_FD && b->type == PO_POLLSET) { - pollset_add_fd(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a); + pollset_add_fd_locked(exec_ctx, (grpc_pollset *)b, (grpc_fd *)a, true); } else if (a->type == PO_POLLSET && b->type == PO_FD) { - pollset_add_fd(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b); + pollset_add_fd_locked(exec_ctx, (grpc_pollset *)a, (grpc_fd *)b, true); } } @@ -1212,7 +1225,14 @@ static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, polling_group *to) { for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { + if (po_cmp(a, b) < 0) { + gpr_mu_lock(&a->mu); gpr_mu_lock(&b->mu); + } else { + GPR_ASSERT(po_cmp(a, b) != 0); + gpr_mu_lock(&b->mu); gpr_mu_lock(&a->mu); + } pg_notify(exec_ctx, a, b); + gpr_mu_unlock(&a->mu); gpr_mu_unlock(&b->mu); } } } @@ -1249,21 +1269,41 @@ static void pg_join(grpc_exec_ctx *exec_ctx, polling_group *pg, /* assumes neither pg nor po are locked; consumes one ref to pg */ pg = pg_lock_latest(pg); /* pg locked */ + for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; + existing != &pg->po; existing = existing->next) { + if (po_cmp(po, existing) < 0) { + gpr_mu_lock(&po->mu); + gpr_mu_lock(&existing->mu); + } else { + GPR_ASSERT(po_cmp(po, existing) != 0); + gpr_mu_lock(&existing->mu); + gpr_mu_lock(&po->mu); + } + /* pg, po, existing locked */ + if (po->group != NULL) { + gpr_mu_unlock(&pg->po.mu); + polling_group *po_group = pg_ref(po->group); + gpr_mu_unlock(&po->mu); + gpr_mu_unlock(&existing->mu); + pg_merge(exec_ctx, pg, po_group); + /* early exit: polling obj picked up a group during joining: we needed + to do a full merge */ + return; + } + pg_notify(exec_ctx, po, existing); + gpr_mu_unlock(&po->mu); + gpr_mu_unlock(&existing->mu); + } gpr_mu_lock(&po->mu); if (po->group != NULL) { gpr_mu_unlock(&pg->po.mu); polling_group *po_group = pg_ref(po->group); gpr_mu_unlock(&po->mu); pg_merge(exec_ctx, pg, po_group); - /* early exit: polling obj picked up a group before joining: we needed + /* early exit: polling obj picked up a group during joining: we needed to do a full merge */ return; } - /* pg, po locked */ - for (polling_obj *existing = pg->po.next /* skip pg - it's just a stub */; - existing != &pg->po; existing = existing->next) { - pg_notify(exec_ctx, po, existing); - } po->group = pg; po->next = &pg->po; po->prev = pg->po.prev; diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index 6f3a51e7175..69e20098d70 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -57,7 +57,7 @@ void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu); * pollset's mutex must be held */ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); -void grpc_pollset_destroy(grpc_pollset *pollset); +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); /* Do some work on a pollset. May involve invoking asynchronous callbacks, or actually polling file From f840110de243ae225f5825c0dda47e9fe89b582c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 09:47:28 -0700 Subject: [PATCH 049/177] Fixup tests --- src/core/lib/iomgr/ev_epoll_linux.c | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 65 +++++++++++-------- src/core/lib/iomgr/ev_poll_posix.c | 2 +- src/core/lib/iomgr/ev_posix.c | 4 +- src/core/lib/iomgr/ev_posix.h | 2 +- .../google_default_credentials.c | 2 +- src/core/lib/surface/alarm.c | 4 +- src/core/lib/surface/call.c | 2 +- src/core/lib/surface/completion_queue.c | 19 +++--- src/core/lib/surface/completion_queue.h | 12 ++-- src/core/lib/surface/server.c | 2 +- .../end2end/fixtures/http_proxy_fixture.c | 2 +- test/core/http/httpcli_test.c | 2 +- test/core/http/httpscli_test.c | 2 +- test/core/iomgr/endpoint_pair_test.c | 2 +- test/core/iomgr/ev_epoll_linux_test.c | 2 +- test/core/iomgr/fd_posix_test.c | 2 +- test/core/iomgr/pollset_set_test.c | 2 +- test/core/iomgr/resolve_address_posix_test.c | 2 +- test/core/iomgr/resolve_address_test.c | 2 +- test/core/iomgr/tcp_client_posix_test.c | 2 +- test/core/iomgr/tcp_posix_test.c | 2 +- test/core/iomgr/tcp_server_posix_test.c | 2 +- test/core/iomgr/udp_server_test.c | 2 +- test/core/security/secure_endpoint_test.c | 2 +- .../surface/concurrent_connectivity_test.c | 2 +- test/core/util/port_server_client.c | 2 +- test/core/util/test_tcp_server.c | 9 ++- 28 files changed, 90 insertions(+), 67 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 766891169ff..0f1bd833f75 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1333,7 +1333,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. So other * than destroying the mutexes, there is nothing special that needs to be done * here */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); gpr_mu_destroy(&pollset->po.mu); } diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index fdde21756bf..f0b9ee39f65 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -265,7 +265,8 @@ static gpr_mu fd_freelist_mu; #ifdef GRPC_FD_REF_COUNT_DEBUG #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(ec, fd, n, reason) unref_by(ec, fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(ec, fd, n, reason) \ + unref_by(ec, fd, n, reason, __FILE__, __LINE__) static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, int line) { gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, @@ -280,23 +281,23 @@ static void ref_by(grpc_fd *fd, int n) { } static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { -grpc_fd *fd = arg; - /* Add the fd to the freelist */ - grpc_iomgr_unregister_object(&fd->iomgr_object); - pollable_destroy(&fd->pollable); - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; + grpc_fd *fd = arg; + /* Add the fd to the freelist */ + grpc_iomgr_unregister_object(&fd->iomgr_object); + pollable_destroy(&fd->pollable); + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); - gpr_mu_unlock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); } #ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, const char *reason, const char *file, - int line) { +static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n, + const char *reason, const char *file, int line) { gpr_atm old; gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), @@ -307,7 +308,9 @@ static void unref_by(grpc_exec_ctx *exec_ctx, grpc_fd *fd, int n) { #endif old = gpr_atm_full_fetch_add(&fd->refst, -n); if (old == n) { - grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); + grpc_closure_sched(exec_ctx, grpc_closure_create(fd_destroy, fd, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); } else { GPR_ASSERT(old > n); } @@ -660,7 +663,8 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { return error; } -static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { +static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, + grpc_pollset_worker *specific_worker) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p " @@ -711,7 +715,6 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_p gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } - } /* p->po.mu must be held before calling this function */ @@ -816,7 +819,8 @@ static bool pollset_is_pollable_fd(grpc_pollset *pollset, pollable *p) { static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { pollable_destroy(&pollset->pollable); if (pollset_is_pollable_fd(pollset, pollset->current_pollable)) { - UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, "pollset_pollable"); + UNREF_BY(exec_ctx, (grpc_fd *)pollset->current_pollable, 2, + "pollset_pollable"); } } @@ -967,7 +971,8 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, return pollset->shutdown_closure == NULL; } -static void end_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, +static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { worker_remove(&pollset->root_worker, PWL_POLLSET, worker); if (NEW_ROOT == @@ -1027,21 +1032,23 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); } - end_worker(pollset, &worker, worker_hdl); + end_worker(exec_ctx, pollset, &worker, worker_hdl); if (worker.pollable != &pollset->pollable) { gpr_mu_unlock(&worker.pollable->po.mu); } return error; } -static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { +static void unref_fd_no_longer_poller(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { grpc_fd *fd = arg; UNREF_BY(exec_ctx, fd, 2, "pollset_pollable"); } /* expects pollsets locked, flag whether fd is locked or not */ -static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd, bool fd_locked) { +static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_fd *fd, + bool fd_locked) { static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; if (pollset->current_pollable == &g_empty_pollable) { @@ -1063,7 +1070,10 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, grpc_pollset * pollable_add_fd(&pollset->pollable, had_fd); pollable_add_fd(&pollset->pollable, fd); } - grpc_closure_sched(exec_ctx, grpc_closure_create(unref_fd_no_longer_poller, had_fd, grpc_schedule_on_exec_ctx), GRPC_ERROR_NONE); + grpc_closure_sched(exec_ctx, + grpc_closure_create(unref_fd_no_longer_poller, had_fd, + grpc_schedule_on_exec_ctx), + GRPC_ERROR_NONE); } return error; } @@ -1226,13 +1236,16 @@ static void pg_broadcast(grpc_exec_ctx *exec_ctx, polling_group *from, for (polling_obj *a = from->po.next; a != &from->po; a = a->next) { for (polling_obj *b = to->po.next; b != &to->po; b = b->next) { if (po_cmp(a, b) < 0) { - gpr_mu_lock(&a->mu); gpr_mu_lock(&b->mu); + gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); } else { GPR_ASSERT(po_cmp(a, b) != 0); - gpr_mu_lock(&b->mu); gpr_mu_lock(&a->mu); + gpr_mu_lock(&b->mu); + gpr_mu_lock(&a->mu); } pg_notify(exec_ctx, a, b); - gpr_mu_unlock(&a->mu); gpr_mu_unlock(&b->mu); + gpr_mu_unlock(&a->mu); + gpr_mu_unlock(&b->mu); } } } diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index b13ec2cbc1e..a83bad319ac 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -817,7 +817,7 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->pollset_set_count = 0; } -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail); while (pollset->local_wakeup_cache) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 620bbde65df..a01de93e9b0 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -204,8 +204,8 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_event_engine->pollset_shutdown(exec_ctx, pollset, closure); } -void grpc_pollset_destroy(grpc_pollset *pollset) { - g_event_engine->pollset_destroy(pollset); +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { + g_event_engine->pollset_destroy(exec_ctx, pollset); } grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index e84a86082ab..59832aaf24a 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -66,7 +66,7 @@ typedef struct grpc_event_engine_vtable { void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu); void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure); - void (*pollset_destroy)(grpc_pollset *pollset); + void (*pollset_destroy)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset); grpc_error *(*pollset_work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker, gpr_timespec now, gpr_timespec deadline); diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c index 97501e6788c..4d8c451ea80 100644 --- a/src/core/lib/security/credentials/google_default/google_default_credentials.c +++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c @@ -99,7 +99,7 @@ static void on_compute_engine_detection_http_response(grpc_exec_ctx *exec_ctx, } static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *e) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static int is_stack_running_on_compute_engine(grpc_exec_ctx *exec_ctx) { diff --git a/src/core/lib/surface/alarm.c b/src/core/lib/surface/alarm.c index e71c0ebfc57..b72d534b7ef 100644 --- a/src/core/lib/surface/alarm.c +++ b/src/core/lib/surface/alarm.c @@ -81,7 +81,9 @@ void grpc_alarm_cancel(grpc_alarm *alarm) { } void grpc_alarm_destroy(grpc_alarm *alarm) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_alarm_cancel(alarm); - GRPC_CQ_INTERNAL_UNREF(alarm->cq, "alarm"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, alarm->cq, "alarm"); gpr_free(alarm); + grpc_exec_ctx_finish(&exec_ctx); } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 3e96d097985..6bd481d3bc6 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -509,7 +509,7 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, } } if (c->cq) { - GRPC_CQ_INTERNAL_UNREF(c->cq, "bind"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, c->cq, "bind"); } get_final_status(call, set_status_value_directly, &c->final_info.final_status, diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 35e9f7eb308..366842d1214 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -182,20 +182,21 @@ void grpc_cq_internal_ref(grpc_completion_queue *cc) { static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_completion_queue *cc = arg; - GRPC_CQ_INTERNAL_UNREF(cc, "pollset_destroy"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, cc, "pollset_destroy"); } #ifdef GRPC_CQ_REF_COUNT_DEBUG -void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason, - const char *file, int line) { +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, + const char *reason, const char *file, int line) { gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "CQ:%p unref %d -> %d %s", cc, (int)cc->owning_refs.count, (int)cc->owning_refs.count - 1, reason); #else -void grpc_cq_internal_unref(grpc_completion_queue *cc) { +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, + grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_destroy(POLLSET_FROM_CQ(cc)); + grpc_pollset_destroy(exec_ctx, POLLSET_FROM_CQ(cc)); #ifndef NDEBUG gpr_free(cc->outstanding_tags); #endif @@ -469,7 +470,7 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, is_finished_arg.first_loop = false; } GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); - GRPC_CQ_INTERNAL_UNREF(cc, "next"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "next"); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(is_finished_arg.stolen_completion == NULL); @@ -664,7 +665,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, } done: GRPC_SURFACE_TRACE_RETURNED_EVENT(cc, &ret); - GRPC_CQ_INTERNAL_UNREF(cc, "pluck"); + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "pluck"); grpc_exec_ctx_finish(&exec_ctx); GPR_ASSERT(is_finished_arg.stolen_completion == NULL); @@ -701,7 +702,9 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { GRPC_API_TRACE("grpc_completion_queue_destroy(cc=%p)", 1, (cc)); GPR_TIMER_BEGIN("grpc_completion_queue_destroy", 0); grpc_completion_queue_shutdown(cc); - GRPC_CQ_INTERNAL_UNREF(cc, "destroy"); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + GRPC_CQ_INTERNAL_UNREF(&exec_ctx, cc, "destroy"); + grpc_exec_ctx_finish(&exec_ctx); GPR_TIMER_END("grpc_completion_queue_destroy", 0); } diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index 1ff3d64293a..a2e7b2d6a49 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -65,17 +65,17 @@ typedef struct grpc_cq_completion { #ifdef GRPC_CQ_REF_COUNT_DEBUG void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason, const char *file, int line); -void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason, - const char *file, int line); +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, + const char *reason, const char *file, int line); #define GRPC_CQ_INTERNAL_REF(cc, reason) \ grpc_cq_internal_ref(cc, reason, __FILE__, __LINE__) -#define GRPC_CQ_INTERNAL_UNREF(cc, reason) \ - grpc_cq_internal_unref(cc, reason, __FILE__, __LINE__) +#define GRPC_CQ_INTERNAL_UNREF(ec, cc, reason) \ + grpc_cq_internal_unref(ec, cc, reason, __FILE__, __LINE__) #else void grpc_cq_internal_ref(grpc_completion_queue *cc); -void grpc_cq_internal_unref(grpc_completion_queue *cc); +void grpc_cq_internal_unref(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc); #define GRPC_CQ_INTERNAL_REF(cc, reason) grpc_cq_internal_ref(cc) -#define GRPC_CQ_INTERNAL_UNREF(cc, reason) grpc_cq_internal_unref(cc) +#define GRPC_CQ_INTERNAL_UNREF(ec, cc, reason) grpc_cq_internal_unref(ec, cc) #endif /* Flag that an operation is beginning: the completion channel will not finish diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 38800ea37b3..b0328cec984 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -408,7 +408,7 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) { request_matcher_destroy(&server->unregistered_request_matcher); } for (i = 0; i < server->cq_count; i++) { - GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server"); + GRPC_CQ_INTERNAL_UNREF(exec_ctx, server->cqs[i], "server"); if (server->started) { gpr_stack_lockfree_destroy(server->request_freelist_per_cq[i]); gpr_free(server->requested_calls_per_cq[i]); diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index 451ed268d30..259abfd151e 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -469,7 +469,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { static void destroy_pollset(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { grpc_pollset* pollset = arg; - grpc_pollset_destroy(pollset); + grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); } diff --git a/test/core/http/httpcli_test.c b/test/core/http/httpcli_test.c index d3b45c45056..21135ddf6b1 100644 --- a/test/core/http/httpcli_test.c +++ b/test/core/http/httpcli_test.c @@ -155,7 +155,7 @@ static void test_post(int port) { } static void destroy_pops(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(grpc_polling_entity_pollset(p)); + grpc_pollset_destroy(exec_ctx, grpc_polling_entity_pollset(p)); } int main(int argc, char **argv) { diff --git a/test/core/http/httpscli_test.c b/test/core/http/httpscli_test.c index acc94091f4e..73eaae87d75 100644 --- a/test/core/http/httpscli_test.c +++ b/test/core/http/httpscli_test.c @@ -157,7 +157,7 @@ static void test_post(int port) { } static void destroy_pops(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(grpc_polling_entity_pollset(p)); + grpc_pollset_destroy(exec_ctx, grpc_polling_entity_pollset(p)); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index c8a60776b9c..4561c3846e1 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -70,7 +70,7 @@ static grpc_endpoint_test_config configs[] = { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 0856023b14f..7693b5a3982 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -113,7 +113,7 @@ static void test_pollset_init(test_pollset *pollsets, int num_pollsets) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static void test_pollset_cleanup(grpc_exec_ctx *exec_ctx, diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index 81d2692a084..d6cee8e43a7 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -535,7 +535,7 @@ static void test_grpc_fd_change(void) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 3a9d459579f..469551c827c 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -86,7 +86,7 @@ static void init_test_pollsets(test_pollset *pollsets, const int num_pollsets) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } static void cleanup_test_pollsets(grpc_exec_ctx *exec_ctx, diff --git a/test/core/iomgr/resolve_address_posix_test.c b/test/core/iomgr/resolve_address_posix_test.c index fa88aca431f..7fba0b92bec 100644 --- a/test/core/iomgr/resolve_address_posix_test.c +++ b/test/core/iomgr/resolve_address_posix_test.c @@ -81,7 +81,7 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) { grpc_pollset_shutdown(exec_ctx, args->pollset, &do_nothing_cb); // exec_ctx needs to be flushed before calling grpc_pollset_destroy() grpc_exec_ctx_flush(exec_ctx); - grpc_pollset_destroy(args->pollset); + grpc_pollset_destroy(exec_ctx, args->pollset); gpr_free(args->pollset); } diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index ea79adc0903..0425c3b3f5a 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -76,7 +76,7 @@ void args_finish(grpc_exec_ctx *exec_ctx, args_struct *args) { grpc_pollset_shutdown(exec_ctx, args->pollset, &do_nothing_cb); // exec_ctx needs to be flushed before calling grpc_pollset_destroy() grpc_exec_ctx_flush(exec_ctx); - grpc_pollset_destroy(args->pollset); + grpc_pollset_destroy(exec_ctx, args->pollset); gpr_free(args->pollset); } diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c index 2fae6774e86..6e1bb43eb5b 100644 --- a/test/core/iomgr/tcp_client_posix_test.c +++ b/test/core/iomgr/tcp_client_posix_test.c @@ -197,7 +197,7 @@ void test_fails(void) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index 2c53a003d23..a1c54e19c1c 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -562,7 +562,7 @@ static grpc_endpoint_test_config configs[] = { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c index 112743b95b7..88aead1dd0d 100644 --- a/test/core/iomgr/tcp_server_posix_test.c +++ b/test/core/iomgr/tcp_server_posix_test.c @@ -444,7 +444,7 @@ static void test_connect(size_t num_connects, static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index 1f1696a7a7a..ee78d6b4adf 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -307,7 +307,7 @@ static void test_receive(int number_of_clients) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index 71d8057ac31..cd6ff2ceacf 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -185,7 +185,7 @@ static void test_leftover(grpc_endpoint_test_config config, size_t slice_size) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c index d6841ea1f88..f0e3394b2e7 100644 --- a/test/core/surface/concurrent_connectivity_test.c +++ b/test/core/surface/concurrent_connectivity_test.c @@ -162,7 +162,7 @@ void bad_server_thread(void *vargs) { static void done_pollset_shutdown(grpc_exec_ctx *exec_ctx, void *pollset, grpc_error *error) { - grpc_pollset_destroy(pollset); + grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 38054dd1e74..2cd50a4ff79 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -58,7 +58,7 @@ typedef struct freereq { static void destroy_pops_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { grpc_pollset *pollset = grpc_polling_entity_pollset(p); - grpc_pollset_destroy(pollset); + grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); grpc_shutdown(); } diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c index 496e579bc39..19086980092 100644 --- a/test/core/util/test_tcp_server.c +++ b/test/core/util/test_tcp_server.c @@ -106,6 +106,10 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) { } static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} +static void finish_pollset(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_pollset_destroy(exec_ctx, arg); +} void test_tcp_server_destroy(test_tcp_server *server) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; @@ -120,9 +124,10 @@ void test_tcp_server_destroy(test_tcp_server *server) { gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), shutdown_deadline) < 0) { test_tcp_server_poll(server, 1); } - grpc_pollset_shutdown(&exec_ctx, server->pollset, &do_nothing_cb); + grpc_pollset_shutdown(&exec_ctx, server->pollset, + grpc_closure_create(finish_pollset, server->pollset, + grpc_schedule_on_exec_ctx)); grpc_exec_ctx_finish(&exec_ctx); - grpc_pollset_destroy(server->pollset); gpr_free(server->pollset); grpc_shutdown(); } From 79d24fb8eb6a6dfe51aaea6de4bf388ba55bdde6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 19:35:19 +0000 Subject: [PATCH 050/177] Fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 22 ++++++++++++++++++---- test/core/util/port_server_client.c | 6 +++--- test/cpp/microbenchmarks/bm_pollset.cc | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index f0b9ee39f65..31316cfc4dc 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -161,6 +161,7 @@ struct grpc_fd { /* The fd is either closed or we relinquished control of it. In either cases, this indicates that the 'fd' on this structure is no longer valid */ + gpr_mu orphaned_mu; bool orphaned; gpr_atm read_closure; @@ -285,6 +286,7 @@ static void fd_destroy(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { /* Add the fd to the freelist */ grpc_iomgr_unregister_object(&fd->iomgr_object); pollable_destroy(&fd->pollable); + gpr_mu_destroy(&fd->orphaned_mu); gpr_mu_lock(&fd_freelist_mu); fd->freelist_next = fd_freelist; fd_freelist = fd; @@ -347,6 +349,7 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; + gpr_mu_init(&new_fd->orphaned_mu); new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); @@ -374,11 +377,11 @@ static grpc_fd *fd_create(int fd, const char *name) { static int fd_wrapped_fd(grpc_fd *fd) { int ret_fd = -1; - gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->orphaned_mu); if (!fd->orphaned) { ret_fd = fd->fd; } - gpr_mu_unlock(&fd->pollable.po.mu); + gpr_mu_unlock(&fd->orphaned_mu); return ret_fd; } @@ -390,6 +393,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *error = GRPC_ERROR_NONE; gpr_mu_lock(&fd->pollable.po.mu); + gpr_mu_lock(&fd->orphaned_mu); fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file @@ -413,6 +417,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + gpr_mu_unlock(&fd->orphaned_mu); gpr_mu_unlock(&fd->pollable.po.mu); UNREF_BY(exec_ctx, fd, 2, reason); /* Drop the reference */ GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); @@ -545,9 +550,11 @@ static void pollable_init(pollable *p, polling_obj_type type) { } static void pollable_destroy(pollable *p) { - close(p->epfd); - grpc_wakeup_fd_destroy(&p->wakeup); po_destroy(&p->po); + if (p->epfd != -1) { + close(p->epfd); + grpc_wakeup_fd_destroy(&p->wakeup); + } } /* ensure that p->epfd, p->wakeup are initialized; p->po.mu must be held */ @@ -590,7 +597,13 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { grpc_error *error = GRPC_ERROR_NONE; static const char *err_desc = "pollable_add_fd"; const int epfd = p->epfd; + GPR_ASSERT(epfd != -1); + gpr_mu_lock(&fd->orphaned_mu); + if (fd->orphaned) { + gpr_mu_unlock(&fd->orphaned_mu); + return GRPC_ERROR_NONE; + } struct epoll_event ev_fd = { .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { @@ -614,6 +627,7 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); } } + gpr_mu_unlock(&fd->orphaned_mu); return error; } diff --git a/test/core/util/port_server_client.c b/test/core/util/port_server_client.c index 2cd50a4ff79..a388d5f27b6 100644 --- a/test/core/util/port_server_client.c +++ b/test/core/util/port_server_client.c @@ -60,7 +60,6 @@ static void destroy_pops_and_shutdown(grpc_exec_ctx *exec_ctx, void *p, grpc_pollset *pollset = grpc_polling_entity_pollset(p); grpc_pollset_destroy(exec_ctx, pollset); gpr_free(pollset); - grpc_shutdown(); } static void freed_port_from_server(grpc_exec_ctx *exec_ctx, void *arg, @@ -122,12 +121,13 @@ void grpc_free_port_using_server(int port) { gpr_mu_unlock(pr.mu); grpc_httpcli_context_destroy(&exec_ctx, &context); - grpc_exec_ctx_finish(&exec_ctx); grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&pr.pops), shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); gpr_free(path); grpc_http_response_destroy(&rsp); + + grpc_shutdown(); } typedef struct portreq { @@ -239,7 +239,6 @@ int grpc_pick_port_using_server(void) { grpc_closure_create(got_port_from_server, &pr, grpc_schedule_on_exec_ctx), &pr.response); grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); - grpc_exec_ctx_finish(&exec_ctx); gpr_mu_lock(pr.mu); while (pr.port == -1) { grpc_pollset_worker *worker = NULL; @@ -258,6 +257,7 @@ int grpc_pick_port_using_server(void) { grpc_pollset_shutdown(&exec_ctx, grpc_polling_entity_pollset(&pr.pops), shutdown_closure); grpc_exec_ctx_finish(&exec_ctx); + grpc_shutdown(); return pr.port; } diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 2a0e96ddf1d..19b3ba0a3c5 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -59,7 +59,7 @@ extern "C" { auto& force_library_initialization = Library::get(); static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) { - grpc_pollset_destroy(static_cast(ps)); + grpc_pollset_destroy(exec_ctx, static_cast(ps)); } static void BM_CreateDestroyPollset(benchmark::State& state) { From 8a2984c4b66b812a1e8519f32406358e44781536 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 19:48:27 +0000 Subject: [PATCH 051/177] add missing unlock --- src/core/lib/iomgr/ev_epollex_linux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 31316cfc4dc..a6fec5d49a1 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -610,6 +610,7 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { switch (errno) { case EEXIST: /* if this fd is already in the epoll set, the workqueue fd must also be - just return */ + gpr_mu_unlock(&fd->orphaned_mu); return GRPC_ERROR_NONE; default: append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); From 144521fc76b2c4af303ffd70e3fcd946866ee994 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 20:19:04 +0000 Subject: [PATCH 052/177] sync fixes --- include/grpc++/server_builder.h | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index d707100a522..a0ba4fc8d28 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,7 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(1), + : num_cqs(gpr_cpu_num_cores()), min_pollers(1), max_pollers(INT_MAX), cq_timeout_msec(1000) {} diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index a6fec5d49a1..af2908f16d4 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -972,7 +972,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (pollset->root_worker != worker) { + while (worker->pollable->root_worker != worker) { if (gpr_cv_wait(&worker->cv, &pollset->current_pollable->po.mu, deadline)) { return false; From e3a69338c38fc63a8a7e3d0e61802b9dff13439b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 18 Apr 2017 16:45:40 +0000 Subject: [PATCH 053/177] Fix wakeup bugs --- src/core/lib/iomgr/ev_epollex_linux.c | 73 +++++++++++++++++++-------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index af2908f16d4..4f221a5f22f 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -666,6 +666,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { grpc_pollset_worker *worker = pollset->root_worker; do { if (worker->initialized_cv) { + worker->kicked = true; gpr_cv_signal(&worker->cv); } else { append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), @@ -727,6 +728,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); } + specific_worker->kicked = true; gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } @@ -850,13 +852,16 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, int timeout = poll_deadline_to_millis_timeout(deadline, now); if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll for %dms", pollset, timeout); + gpr_log(GPR_DEBUG, "PS:%p poll %p for %dms", pollset, p, timeout); } if (timeout != 0) { GRPC_SCHEDULING_START_BLOCKING_REGION; } - int r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); + int r; + do { + r = epoll_wait(p->epfd, events, MAX_EPOLL_EVENTS, timeout); + } while (r < 0 && errno == EINTR); if (timeout != 0) { GRPC_SCHEDULING_END_BLOCKING_REGION; } @@ -864,7 +869,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll got %d events", pollset, r); + gpr_log(GPR_DEBUG, "PS:%p poll %p got %d events", pollset, p, r); } grpc_error *error = GRPC_ERROR_NONE; @@ -872,7 +877,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll got global_wakeup_fd", pollset); + gpr_log(GPR_DEBUG, "PS:%p poll %p got global_wakeup_fd", pollset, p); } grpc_timer_consume_kick(); @@ -880,7 +885,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, err_desc); } else if (data_ptr == &p->wakeup) { if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll got pollset_wakeup", pollset); + gpr_log(GPR_DEBUG, "PS:%p poll %p got pollset_wakeup", pollset, p); } append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); } else { @@ -890,11 +895,11 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; if (grpc_polling_trace) { - gpr_log( - GPR_DEBUG, - "PS:%p poll got fd %p(%d/%d): is_wq=%d cancel=%d read=%d write=%d", - pollset, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, is_workqueue, - cancel, read_ev, write_ev); + gpr_log(GPR_DEBUG, + "PS:%p poll %p got fd %p(%d/%d): is_wq=%d cancel=%d read=%d " + "write=%d", + pollset, p, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, + is_workqueue, cancel, read_ev, write_ev); } if (is_workqueue) { append_error(&error, @@ -956,8 +961,9 @@ static worker_remove_result worker_remove(grpc_pollset_worker **root, /* Return true if this thread should poll */ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, - grpc_pollset_worker **worker_hdl, + grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { + bool do_poll = true; if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; worker->kicked = false; @@ -972,18 +978,43 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (!worker_insert(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (worker->pollable->root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &pollset->current_pollable->po.mu, - deadline)) { - return false; - } - if (worker->kicked) { - return false; + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&pollset->pollable.po.mu); + } + if (grpc_polling_trace && worker->pollable->root_worker != worker) { + gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, + worker->pollable, worker, + poll_deadline_to_millis_timeout(deadline, *now)); + } + while (do_poll && worker->pollable->root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, + worker->pollable, worker); + } + do_poll = false; + } else if (worker->kicked) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, + worker); + } + do_poll = false; + } else if (grpc_polling_trace && + worker->pollable->root_worker != worker) { + gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, + worker->pollable, worker); } } + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&worker->pollable->po.mu); + gpr_mu_lock(&pollset->pollable.po.mu); + gpr_mu_lock(&worker->pollable->po.mu); + } + *now = gpr_now(now->clock_type); } - return pollset->shutdown_closure == NULL; + return do_poll && pollset->shutdown_closure == NULL && + pollset->current_pollable == worker->pollable; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -1010,7 +1041,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - if (grpc_polling_trace) { + if (0 && grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, @@ -1026,7 +1057,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (pollset->current_pollable != &pollset->pollable) { gpr_mu_lock(&pollset->current_pollable->po.mu); } - if (begin_worker(pollset, &worker, worker_hdl, deadline)) { + if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); From d37d142a6339ba1ea600fe4c2002971dc8337949 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 18 Apr 2017 19:41:07 +0000 Subject: [PATCH 054/177] sync fixes --- src/core/lib/iomgr/ev_epollex_linux.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4f221a5f22f..d6504f5bace 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -711,11 +711,17 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } return GRPC_ERROR_NONE; } + } else if (specific_worker->kicked) { + if (grpc_polling_trace) { + gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_already_kicked", p); + } + return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); } + specific_worker->kicked = true; return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { if (grpc_polling_trace) { @@ -723,6 +729,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, } grpc_error *err = pollable_materialize(p); if (err != GRPC_ERROR_NONE) return err; + specific_worker->kicked = true; return grpc_wakeup_fd_wakeup(&p->wakeup); } else { if (grpc_polling_trace) { From cc92eb42a4404b92f7dacace68090fd1f9111c4a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 06:54:25 -0700 Subject: [PATCH 055/177] Update to new API --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 9d7f65d2923..0d267da7237 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -67,7 +67,9 @@ static void pollset_init(grpc_pollset* ps, gpr_mu** mu) { *mu = &ps->mu; } -static void pollset_destroy(grpc_pollset* ps) { gpr_mu_destroy(&ps->mu); } +static void pollset_destroy(grpc_exec_ctx* exec_ctx, grpc_pollset* ps) { + gpr_mu_destroy(&ps->mu); +} static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) { return GRPC_ERROR_NONE; From 653ca0b517d8695cc720eda99b98438dbc349d46 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 09:41:30 -0700 Subject: [PATCH 056/177] Add epollex to BUILD --- BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BUILD b/BUILD index 542c43f6321..3f43be980bc 100644 --- a/BUILD +++ b/BUILD @@ -474,6 +474,7 @@ grpc_cc_library( "src/core/lib/iomgr/endpoint_pair_windows.c", "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll_linux.c", + "src/core/lib/iomgr/ev_epollex_linux.c", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_posix.c", "src/core/lib/iomgr/exec_ctx.c", @@ -596,6 +597,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epollex_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", From e107b0d439b3d394b8b60222b6e6afb256e67f5e Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 19 Apr 2017 14:01:13 -0700 Subject: [PATCH 057/177] Add to codegen interface, refactor proto serialization This change allows for some internal proto serialization changes to be made --- include/grpc++/impl/codegen/config_protobuf.h | 2 + include/grpc++/impl/codegen/core_codegen.h | 6 +- .../impl/codegen/core_codegen_interface.h | 6 +- include/grpc++/impl/codegen/proto_utils.h | 107 +++++++++++------- src/cpp/common/core_codegen.cc | 14 +++ 5 files changed, 91 insertions(+), 44 deletions(-) diff --git a/include/grpc++/impl/codegen/config_protobuf.h b/include/grpc++/impl/codegen/config_protobuf.h index 8620d4b2183..e66189f8d12 100644 --- a/include/grpc++/impl/codegen/config_protobuf.h +++ b/include/grpc++/impl/codegen/config_protobuf.h @@ -34,6 +34,8 @@ #ifndef GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H #define GRPCXX_IMPL_CODEGEN_CONFIG_PROTOBUF_H +#define GRPC_OPEN_SOURCE_PROTO + #ifndef GRPC_CUSTOM_PROTOBUF_INT64 #include #define GRPC_CUSTOM_PROTOBUF_INT64 ::google::protobuf::int64 diff --git a/include/grpc++/impl/codegen/core_codegen.h b/include/grpc++/impl/codegen/core_codegen.h index 65151590b25..76cbe2047ac 100644 --- a/include/grpc++/impl/codegen/core_codegen.h +++ b/include/grpc++/impl/codegen/core_codegen.h @@ -79,11 +79,15 @@ class CoreCodegen : public CoreCodegenInterface { grpc_byte_buffer* grpc_raw_byte_buffer_create(grpc_slice* slice, size_t nslices) override; - + grpc_slice grpc_slice_new_with_user_data(void* p, size_t len, + void (*destroy)(void*), + void* user_data) override; grpc_slice grpc_empty_slice() override; grpc_slice grpc_slice_malloc(size_t length) override; void grpc_slice_unref(grpc_slice slice) override; + grpc_slice grpc_slice_ref(grpc_slice slice) override; grpc_slice grpc_slice_split_tail(grpc_slice* s, size_t split) override; + grpc_slice grpc_slice_split_head(grpc_slice* s, size_t split) override; void grpc_slice_buffer_add(grpc_slice_buffer* sb, grpc_slice slice) override; void grpc_slice_buffer_pop(grpc_slice_buffer* sb) override; grpc_slice grpc_slice_from_static_buffer(const void* buffer, diff --git a/include/grpc++/impl/codegen/core_codegen_interface.h b/include/grpc++/impl/codegen/core_codegen_interface.h index 529bef687bb..fc25a641b5c 100644 --- a/include/grpc++/impl/codegen/core_codegen_interface.h +++ b/include/grpc++/impl/codegen/core_codegen_interface.h @@ -95,11 +95,15 @@ class CoreCodegenInterface { virtual grpc_byte_buffer* grpc_raw_byte_buffer_create(grpc_slice* slice, size_t nslices) = 0; - + virtual grpc_slice grpc_slice_new_with_user_data(void* p, size_t len, + void (*destroy)(void*), + void* user_data) = 0; virtual grpc_slice grpc_empty_slice() = 0; virtual grpc_slice grpc_slice_malloc(size_t length) = 0; virtual void grpc_slice_unref(grpc_slice slice) = 0; + virtual grpc_slice grpc_slice_ref(grpc_slice slice) = 0; virtual grpc_slice grpc_slice_split_tail(grpc_slice* s, size_t split) = 0; + virtual grpc_slice grpc_slice_split_head(grpc_slice* s, size_t split) = 0; virtual void grpc_slice_buffer_add(grpc_slice_buffer* sb, grpc_slice slice) = 0; virtual void grpc_slice_buffer_pop(grpc_slice_buffer* sb) = 0; diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h index 6df9de4fd22..491b8eb9434 100644 --- a/include/grpc++/impl/codegen/proto_utils.h +++ b/include/grpc++/impl/codegen/proto_utils.h @@ -54,8 +54,7 @@ class GrpcBufferWriterPeer; const int kGrpcBufferWriterMaxBufferLength = 8192; -class GrpcBufferWriter final - : public ::grpc::protobuf::io::ZeroCopyOutputStream { +class GrpcBufferWriter : public ::grpc::protobuf::io::ZeroCopyOutputStream { public: explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size) : block_size_(block_size), byte_count_(0), have_backup_(false) { @@ -103,6 +102,8 @@ class GrpcBufferWriter final grpc::protobuf::int64 ByteCount() const override { return byte_count_; } + grpc_slice_buffer* SliceBuffer() { return slice_buffer_; } + private: friend class GrpcBufferWriterPeer; const int block_size_; @@ -113,8 +114,7 @@ class GrpcBufferWriter final grpc_slice slice_; }; -class GrpcBufferReader final - : public ::grpc::protobuf::io::ZeroCopyInputStream { +class GrpcBufferReader : public ::grpc::protobuf::io::ZeroCopyInputStream { public: explicit GrpcBufferReader(grpc_byte_buffer* buffer) : byte_count_(0), backup_count_(0), status_() { @@ -175,64 +175,87 @@ class GrpcBufferReader final return byte_count_ - backup_count_; } - private: + protected: int64_t byte_count_; int64_t backup_count_; grpc_byte_buffer_reader reader_; grpc_slice slice_; Status status_; }; + +// BufferWriter must be a subclass of io::ZeroCopyOutputStream. +template +Status GenericSerialize(const grpc::protobuf::Message& msg, + grpc_byte_buffer** bp, bool* own_buffer) { + *own_buffer = true; + int byte_size = msg.ByteSize(); + if (byte_size <= internal::kGrpcBufferWriterMaxBufferLength) { + grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size); + GPR_CODEGEN_ASSERT( + GRPC_SLICE_END_PTR(slice) == + msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice))); + *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1); + g_core_codegen_interface->grpc_slice_unref(slice); + return g_core_codegen_interface->ok(); + } else { + BufferWriter writer(bp, internal::kGrpcBufferWriterMaxBufferLength); + return msg.SerializeToZeroCopyStream(&writer) + ? g_core_codegen_interface->ok() + : Status(StatusCode::INTERNAL, "Failed to serialize message"); + } +} + +// BufferReader must be a subclass of io::ZeroCopyInputStream. +template +Status GenericDeserialize(grpc_byte_buffer* buffer, + grpc::protobuf::Message* msg) { + if (buffer == nullptr) { + return Status(StatusCode::INTERNAL, "No payload"); + } + Status result = g_core_codegen_interface->ok(); + { + BufferReader reader(buffer); + if (!reader.status().ok()) { + return reader.status(); + } + ::grpc::protobuf::io::CodedInputStream decoder(&reader); + decoder.SetTotalBytesLimit(INT_MAX, INT_MAX); + if (!msg->ParseFromCodedStream(&decoder)) { + result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); + } + if (!decoder.ConsumedEntireMessage()) { + result = Status(StatusCode::INTERNAL, "Did not read entire message"); + } + } + g_core_codegen_interface->grpc_byte_buffer_destroy(buffer); + return result; +} + } // namespace internal +// this is needed so the following class does not conflict with protobuf +// serializers that utilize internal-only tools. +#ifdef GRPC_OPEN_SOURCE_PROTO +// This class provides a protobuf serializer. It translates between protobuf +// objects and grpc_byte_buffers. More information about SerializationTraits can +// be found in include/grpc++/impl/codegen/serialization_traits.h. template class SerializationTraits::value>::type> { public: static Status Serialize(const grpc::protobuf::Message& msg, grpc_byte_buffer** bp, bool* own_buffer) { - *own_buffer = true; - int byte_size = msg.ByteSize(); - if (byte_size <= internal::kGrpcBufferWriterMaxBufferLength) { - grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size); - GPR_CODEGEN_ASSERT( - GRPC_SLICE_END_PTR(slice) == - msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice))); - *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1); - g_core_codegen_interface->grpc_slice_unref(slice); - return g_core_codegen_interface->ok(); - } else { - internal::GrpcBufferWriter writer( - bp, internal::kGrpcBufferWriterMaxBufferLength); - return msg.SerializeToZeroCopyStream(&writer) - ? g_core_codegen_interface->ok() - : Status(StatusCode::INTERNAL, "Failed to serialize message"); - } + return internal::GenericSerialize( + msg, bp, own_buffer); } static Status Deserialize(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg) { - if (buffer == nullptr) { - return Status(StatusCode::INTERNAL, "No payload"); - } - Status result = g_core_codegen_interface->ok(); - { - internal::GrpcBufferReader reader(buffer); - if (!reader.status().ok()) { - return reader.status(); - } - ::grpc::protobuf::io::CodedInputStream decoder(&reader); - decoder.SetTotalBytesLimit(INT_MAX, INT_MAX); - if (!msg->ParseFromCodedStream(&decoder)) { - result = Status(StatusCode::INTERNAL, msg->InitializationErrorString()); - } - if (!decoder.ConsumedEntireMessage()) { - result = Status(StatusCode::INTERNAL, "Did not read entire message"); - } - } - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer); - return result; + return internal::GenericDeserialize(buffer, + msg); } }; +#endif } // namespace grpc diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc index 0dd758ec4e5..3ece5aadded 100644 --- a/src/cpp/common/core_codegen.cc +++ b/src/cpp/common/core_codegen.cc @@ -116,6 +116,12 @@ grpc_byte_buffer* CoreCodegen::grpc_raw_byte_buffer_create(grpc_slice* slice, return ::grpc_raw_byte_buffer_create(slice, nslices); } +grpc_slice CoreCodegen::grpc_slice_new_with_user_data(void* p, size_t len, + void (*destroy)(void*), + void* user_data) { + return ::grpc_slice_new_with_user_data(p, len, destroy, user_data); +} + grpc_slice CoreCodegen::grpc_empty_slice() { return ::grpc_empty_slice(); } grpc_slice CoreCodegen::grpc_slice_malloc(size_t length) { @@ -126,10 +132,18 @@ void CoreCodegen::grpc_slice_unref(grpc_slice slice) { ::grpc_slice_unref(slice); } +grpc_slice CoreCodegen::grpc_slice_ref(grpc_slice slice) { + return ::grpc_slice_ref(slice); +} + grpc_slice CoreCodegen::grpc_slice_split_tail(grpc_slice* s, size_t split) { return ::grpc_slice_split_tail(s, split); } +grpc_slice CoreCodegen::grpc_slice_split_head(grpc_slice* s, size_t split) { + return ::grpc_slice_split_head(s, split); +} + grpc_slice CoreCodegen::grpc_slice_from_static_buffer(const void* buffer, size_t length) { return ::grpc_slice_from_static_buffer(buffer, length); From 1d15194027649936ab579aeeba81240252e13d2f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 23:22:48 +0000 Subject: [PATCH 058/177] Fix race --- src/core/lib/iomgr/ev_epollex_linux.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d6504f5bace..0839a40429a 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -665,6 +665,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { + gpr_mu_lock(&worker->pollable->po.mu); if (worker->initialized_cv) { worker->kicked = true; gpr_cv_signal(&worker->cv); @@ -672,6 +673,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), "pollset_shutdown"); } + gpr_mu_unlock(&worker->pollable->po.mu); worker = worker->links[PWL_POLLSET].next; } while (worker != pollset->root_worker); From 227e11bdee93bcc88abf664be23aab8cafbaf653 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 18:25:02 +0000 Subject: [PATCH 059/177] Fix missing edge on shutdown path --- src/core/lib/iomgr/ev_epollex_linux.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 0839a40429a..adbdaf6e5b7 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -825,6 +825,7 @@ static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + pollset->shutdown_closure = NULL; } } @@ -1029,7 +1030,6 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - worker_remove(&pollset->root_worker, PWL_POLLSET, worker); if (NEW_ROOT == worker_remove(&worker->pollable->root_worker, PWL_POLLABLE, worker)) { gpr_cv_signal(&worker->pollable->root_worker->cv); @@ -1040,6 +1040,9 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (pollset_is_pollable_fd(pollset, worker->pollable)) { UNREF_BY(exec_ctx, (grpc_fd *)worker->pollable, 2, "one_poll"); } + if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } } /* pollset->po.mu lock must be held by the caller before calling this. From ffe51bd8ad96f5b26c6a09e039e49538a7d140b5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 08:30:24 -0700 Subject: [PATCH 060/177] Fix? uv portability --- src/core/lib/iomgr/pollset_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/pollset_uv.c b/src/core/lib/iomgr/pollset_uv.c index a2f81bcd78d..5923da98e27 100644 --- a/src/core/lib/iomgr/pollset_uv.c +++ b/src/core/lib/iomgr/pollset_uv.c @@ -106,7 +106,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); } -void grpc_pollset_destroy(grpc_pollset *pollset) { +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { uv_close((uv_handle_t *)&pollset->timer, timer_close_cb); // timer.data is a boolean indicating that the timer has finished closing pollset->timer.data = (void *)0; From 0bf244769910fdc9f467bd531c82d58bc7941500 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 08:31:20 -0700 Subject: [PATCH 061/177] Fix? older compiler portability --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index adbdaf6e5b7..4000d0445f5 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -64,7 +64,7 @@ #include "src/core/lib/support/spinlock.h" #ifndef EPOLLEXCLUSIVE -#define EPOLLEXCLUSIVE (1u << 28) +#define EPOLLEXCLUSIVE (1 << 28) #endif /* TODO: sreek: Right now, this wakes up all pollers. In future we should make From c8d9b9c3437ad799a718db171a665b0bc094ad38 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 16:41:34 +0000 Subject: [PATCH 062/177] Fix deadlock --- src/core/lib/iomgr/ev_epollex_linux.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 4000d0445f5..3a0335ace0c 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -665,7 +665,9 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { if (pollset->root_worker != NULL) { grpc_pollset_worker *worker = pollset->root_worker; do { - gpr_mu_lock(&worker->pollable->po.mu); + if (worker->pollable != &pollset->pollable) { + gpr_mu_lock(&worker->pollable->po.mu); + } if (worker->initialized_cv) { worker->kicked = true; gpr_cv_signal(&worker->cv); @@ -673,7 +675,9 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { append_error(&error, grpc_wakeup_fd_wakeup(&worker->pollable->wakeup), "pollset_shutdown"); } - gpr_mu_unlock(&worker->pollable->po.mu); + if (worker->pollable != &pollset->pollable) { + gpr_mu_unlock(&worker->pollable->po.mu); + } worker = worker->links[PWL_POLLSET].next; } while (worker != pollset->root_worker); From 1898a29f7ecf2ac4f706351d68dedd5767f02ec9 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 19 Apr 2017 19:37:39 -0700 Subject: [PATCH 063/177] Fixed pollers --- include/grpc++/server_builder.h | 2 +- src/core/lib/iomgr/ev_epoll_linux.c | 362 +++++++++++++++++++------- test/core/iomgr/ev_epoll_linux_test.c | 46 +++- 3 files changed, 320 insertions(+), 90 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 7ac23349c84..68c5344e2a1 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,7 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(1), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(gpr_cpu_num_cores()), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index e603a755937..16a9199ab92 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -97,6 +97,9 @@ void grpc_use_signal(int signum) { } } +/* The maximum number of polling threads per polling island */ +#define GRPC_MAX_POLLERS_PER_ISLAND 1 + struct polling_island; typedef enum { @@ -195,6 +198,11 @@ static void fd_global_shutdown(void); #endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ +typedef struct worker_node { + struct worker_node *next; + struct worker_node *prev; +} worker_node; + /* This is also used as grpc_workqueue (by directly casing it) */ typedef struct polling_island { grpc_closure_scheduler workqueue_scheduler; @@ -229,6 +237,9 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; + gpr_mu worker_list_mu; + worker_node worker_list_head; + /* The fd of the underlying epoll set */ int epoll_fd; @@ -241,14 +252,21 @@ typedef struct polling_island { /******************************************************************************* * Pollset Declarations */ +#define WORKER_FROM_WORKER_LIST_NODE(p) \ + (struct grpc_pollset_worker *)(((char *)(p)) - \ + offsetof(grpc_pollset_worker, pi_list_link)) struct grpc_pollset_worker { /* Thread id of this worker */ pthread_t pt_id; /* Used to prevent a worker from getting kicked multiple times */ gpr_atm is_kicked; + struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; + + gpr_atm is_polling_turn; + worker_node pi_list_link; }; struct grpc_pollset { @@ -392,7 +410,47 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { } } -/* The caller is expected to hold pi->mu lock before calling this function */ +static void worker_node_init(worker_node *node) { + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void push_back_worker_node(worker_node *head, worker_node *node) { + node->next = head; + node->prev = head->prev; + head->prev->next = node; + head->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void remove_worker_node(worker_node *node) { + node->next->prev = node->prev; + node->prev->next = node->next; + /* If node's next and prev point to itself, the node is considered detached + * from the list*/ + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static worker_node *pop_front_worker_node(worker_node *head) { + worker_node *node = head->next; + if (node != head) { + remove_worker_node(node); + } else { + node = NULL; + } + + return node; +} + +/* Returns true if the node's next and prev are pointing to itself (which + indicates that the node is not in the list */ +static bool is_worker_node_detached(worker_node *node) { + return (node->next == node->prev && node->next == node); +} + +/* The caller is expected to hold pi->mu lock before calling this function + */ static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, size_t fd_count, bool add_fd_refs, grpc_error **error) { @@ -546,6 +604,9 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, gpr_atm_rel_store(&pi->poller_count, 0); gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); + gpr_mu_init(&pi->worker_list_mu); + worker_node_init(&pi->worker_list_head); + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), err_desc)) { goto done; @@ -584,6 +645,9 @@ static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { gpr_mpscq_destroy(&pi->workqueue_items); gpr_mu_destroy(&pi->mu); grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + gpr_mu_destroy(&pi->worker_list_mu); + GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); + gpr_free(pi->fds); gpr_free(pi); } @@ -1102,6 +1166,7 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static __thread bool g_initialized_sigmask; static __thread sigset_t g_orig_sigmask; +static __thread sigset_t g_wakeup_sig_set; static void sig_handler(int sig_num) { #ifdef GRPC_EPOLL_DEBUG @@ -1109,6 +1174,14 @@ static void sig_handler(int sig_num) { #endif } +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->pt_id = pthread_self(); + worker->next = worker->prev = NULL; + gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); + gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); + worker_node_init(&worker->pi_list_link); +} + static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } /* Global state management */ @@ -1125,11 +1198,12 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { +static grpc_error *worker_kick(grpc_pollset_worker *worker, + gpr_atm *is_kicked) { grpc_error *err = GRPC_ERROR_NONE; /* Kick the worker only if it was not already kicked */ - if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { + if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { GRPC_POLLING_TRACE( "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", (void *)worker, (long int)worker->pt_id); @@ -1141,6 +1215,14 @@ static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { return err; } +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_kicked); +} + +static grpc_error *poller_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_polling_turn); +} + /* Return 1 if the pollset has active threads in pollset_work (pollset must * be locked) */ static int pollset_has_workers(grpc_pollset *p) { @@ -1246,6 +1328,22 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->shutdown_done = NULL; } +/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + /* Convert a timespec to milliseconds: - Very small or negative poll times are clamped to zero to do a non-blocking poll (which becomes spin polling) @@ -1364,35 +1462,190 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, return false; } +/* NOTE: May modify 'now' */ +static bool acquire_polling_lease(grpc_pollset_worker *worker, + polling_island *pi, gpr_timespec deadline, + gpr_timespec *now) { + bool is_lease_acquired = false; + + gpr_mu_lock(&pi->worker_list_mu); // Lock + long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + + if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { + push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); + gpr_mu_unlock(&pi->worker_list_mu); // Unlock + + bool is_timeout = false; + int ret; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); + if (timeout_ms == -1) { + ret = sigwaitinfo(&g_wakeup_sig_set, NULL); + } else { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + } + + if (ret == -1) { + if (errno == EAGAIN) { + // gpr_log(GPR_INFO, "timeout"); // TODO: sreek remove this + // log-line + } else { + gpr_log(GPR_ERROR, "Failed with retcode: %d (timeout_ms: %d)", errno, + timeout_ms); + } + is_timeout = true; + } + + bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); + /* + if (is_polling_turn) { +/ gpr_log(GPR_ERROR, "do epoll is true (timeout_ms:%d)", + timeout_ms); // TODO: sreek remove this logline + } + */ + + bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + if (is_kicked || is_timeout) { + *now = deadline; + } else if (is_polling_turn) { + *now = gpr_now(GPR_CLOCK_MONOTONIC); + } + + gpr_mu_lock(&pi->worker_list_mu); // Lock + /* The node might have already been removed from the list by the poller + that kicked this. However it is safe to call 'remove_worker_node' on + an already detached node */ + remove_worker_node(&worker->pi_list_link); + num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + } + + if (num_pollers < GRPC_MAX_POLLERS_PER_ISLAND) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); // Add a poller + is_lease_acquired = true; + } + + gpr_mu_unlock(&pi->worker_list_mu); // Unlock + return is_lease_acquired; +} + +static void release_polling_lease(polling_island *pi, grpc_error **error) { + gpr_mu_lock(&pi->worker_list_mu); // Lock + + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); // Remove poller + worker_node *node = pop_front_worker_node(&pi->worker_list_head); + if (node != NULL) { + grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); + append_error(error, poller_kick(next_worker), "poller kick error"); + } + + gpr_mu_unlock(&pi->worker_list_mu); +} + #define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + if (!acquire_polling_lease(worker, pi, deadline, &now)) { + return; + } + + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = + epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + release_polling_lease(pi, error); + + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + /* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, int timeout_ms, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, sigset_t *sig_mask, grpc_error **error) { - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int epoll_fd = -1; - int ep_rv; polling_island *pi = NULL; - char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the latest polling island pointed by pollset->po.pi - Since epoll_fd is immutable, we can read it without obtaining the polling - island lock. There is however a possibility that the polling island (from - which we got the epoll_fd) got merged with another island while we are - in this function. This is still okay because in such a case, we will wakeup - right-away from epoll_wait() and pick up the latest polling_island the next - this function (i.e pollset_work_and_unlock()) is called */ + Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. There is however a possibility that the polling island from + which we got the epoll_fd, got merged with another island in the meantime. + This is okay because in such a case, we will wakeup right-away from + epoll_pwait() (because any merge will poison the old polling island's epoll + set 'polling_island_wakeup_fd') and then pick up the latest polling_island + the next time this function - pollset_work_and_unlock()) is called */ if (pollset->po.pi == NULL) { pollset->po.pi = polling_island_create(exec_ctx, NULL, error); if (pollset->po.pi == NULL) { GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. We cannot continue */ + return; /* Fatal error. Cannot continue */ } PI_ADD_REF(pollset->po.pi, "ps"); @@ -1423,70 +1676,10 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ if (!maybe_do_workqueue_work(exec_ctx, pi)) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); g_current_thread_polling_island = pi; - - GRPC_SCHEDULING_START_BLOCKING_REGION; - ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, - sig_mask); - GRPC_SCHEDULING_END_BLOCKING_REGION; - if (ep_rv < 0) { - if (errno != EINTR) { - gpr_asprintf(&err_msg, - "epoll_wait() epoll fd: %d failed with error: %d (%s)", - epoll_fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - } else { - /* We were interrupted. Save an interation by doing a zero timeout - epoll_wait to see if there are any other events of interest */ - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p received kick", - (void *)pollset, (void *)worker); - ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - } - } - -#ifdef GRPC_TSAN - /* See the definition of g_poll_sync for more details */ - gpr_atm_acq_load(&g_epoll_sync); -#endif /* defined(GRPC_TSAN) */ - - for (int i = 0; i < ep_rv; ++i) { - void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { - append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), - err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " - "%d) got merged", - (void *)pollset, (void *)worker, epoll_fd); - /* This means that our polling island is merged with a different - island. We do not have to do anything here since the subsequent call - to the function pollset_work_and_unlock() will pick up the correct - epoll_fd */ - } else { - grpc_fd *fd = data_ptr; - int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write_ev = ep_ev[i].events & EPOLLOUT; - if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); - } - if (write_ev || cancel) { - fd_become_writable(exec_ctx, fd); - } - } - } - + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, + deadline, sig_mask, error); g_current_thread_polling_island = NULL; - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); } GPR_ASSERT(pi != NULL); @@ -1510,14 +1703,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec now, gpr_timespec deadline) { GPR_TIMER_BEGIN("pollset_work", 0); grpc_error *error = GRPC_ERROR_NONE; - int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); - - sigset_t new_mask; grpc_pollset_worker worker; - worker.next = worker.prev = NULL; - worker.pt_id = pthread_self(); - gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); + pollset_worker_init(&worker); if (worker_hdl) *worker_hdl = &worker; @@ -1551,9 +1739,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, misses acting on a kick */ if (!g_initialized_sigmask) { - sigemptyset(&new_mask); - sigaddset(&new_mask, grpc_wakeup_signal); - pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); + sigemptyset(&g_wakeup_sig_set); + sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); sigdelset(&g_orig_sigmask, grpc_wakeup_signal); g_initialized_sigmask = true; /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. @@ -1568,7 +1756,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, push_front_worker(pollset, &worker); /* Add worker to pollset */ - pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms, + pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, &g_orig_sigmask, &error); grpc_exec_ctx_flush(exec_ctx); diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 0856023b14f..f7745cddb4e 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -38,7 +38,10 @@ #include "src/core/lib/iomgr/ev_posix.h" #include +#include +#include #include +#include #include #include @@ -327,7 +330,7 @@ static __thread int thread_wakeups = 0; static void test_threading_loop(void *arg) { threading_shared *shared = arg; - while (thread_wakeups < 1000000) { + while (thread_wakeups < 20000) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker; gpr_mu_lock(shared->mu); @@ -360,7 +363,7 @@ static void test_threading(void) { shared.pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(shared.pollset, &shared.mu); - gpr_thd_id thds[10]; + gpr_thd_id thds[20]; for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); @@ -399,6 +402,44 @@ static void test_threading(void) { gpr_free(shared.pollset); } +/* Convert milliseconds into 'struct timespec' struct. millis == -1 is + * * considered as an infinity-time in future */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + +void test_sigwait() { + sigset_t wakeup_sig_set; + sigemptyset(&wakeup_sig_set); + sigaddset(&wakeup_sig_set, SIGRTMIN + 6); + int timeout_ms[] = {10, 1400}; + + for (size_t i = 0; i < GPR_ARRAY_SIZE(timeout_ms); i++) { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms[i]); + gpr_log(GPR_ERROR, "sigwait_timeout: %ld, %ld", sigwait_timeout.tv_sec, + sigwait_timeout.tv_nsec); + + gpr_log(GPR_ERROR, "Waiting for %d ms...", timeout_ms[i]); + gpr_timespec bef = gpr_now(GPR_CLOCK_REALTIME); + sigtimedwait(&wakeup_sig_set, NULL, &sigwait_timeout); + gpr_timespec af = gpr_now(GPR_CLOCK_REALTIME); + + gpr_log(GPR_ERROR, "Bef: %ld, %d", bef.tv_sec, bef.tv_nsec); + gpr_log(GPR_ERROR, "Aft: %ld, %d", af.tv_sec, af.tv_nsec); + } +} + int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); @@ -409,6 +450,7 @@ int main(int argc, char **argv) { test_add_fd_to_pollset(); test_pollset_queue_merge_items(); test_threading(); + test_sigwait(); } else { gpr_log(GPR_INFO, "Skipping the test. The test is only relevant for 'epoll' " From 92290167d63fd8b591e0028ba89a14546783b6b1 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 21 Apr 2017 15:40:27 -0700 Subject: [PATCH 064/177] Minor changes --- include/grpc++/server_builder.h | 5 +++- src/core/lib/iomgr/ev_epoll_linux.c | 41 +++++++++++++++++------------ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 68c5344e2a1..a56f81dc4b3 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,10 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(gpr_cpu_num_cores()), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(gpr_cpu_num_cores()), + min_pollers(1), + max_pollers(2), + cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 16a9199ab92..a686d9e0ac3 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -237,6 +237,7 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; + /* The list of workers waiting to do polling on this polling island */ gpr_mu worker_list_mu; worker_node worker_list_head; @@ -265,7 +266,10 @@ struct grpc_pollset_worker { struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; + /* Indicates if it is this worker's turn to do epoll */ gpr_atm is_polling_turn; + + /* Node in the polling island's worker list. */ worker_node pi_list_link; }; @@ -1468,12 +1472,12 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, gpr_timespec *now) { bool is_lease_acquired = false; - gpr_mu_lock(&pi->worker_list_mu); // Lock + gpr_mu_lock(&pi->worker_list_mu); // LOCK long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); - gpr_mu_unlock(&pi->worker_list_mu); // Unlock + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK bool is_timeout = false; int ret; @@ -1487,51 +1491,54 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, if (ret == -1) { if (errno == EAGAIN) { - // gpr_log(GPR_INFO, "timeout"); // TODO: sreek remove this - // log-line + is_timeout = true; } else { + /* TODO: sreek This should not happen. If we see these log messages, it + * means we are most likely something incorrect in the setup needed for + * sigwaitinfo/sigtimedwait */ gpr_log(GPR_ERROR, "Failed with retcode: %d (timeout_ms: %d)", errno, timeout_ms); } - is_timeout = true; } + /* Did the worker come out of sigtimedwait due to a thread that just + * exited epoll and kicking it (in release_polling_lease function). */ bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); - /* - if (is_polling_turn) { -/ gpr_log(GPR_ERROR, "do epoll is true (timeout_ms:%d)", - timeout_ms); // TODO: sreek remove this logline - } - */ + /* Did the worker come out of sigtimedwait due to a thread alerting it that + * some completion event was (likely) available in the completion queue */ bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + if (is_kicked || is_timeout) { - *now = deadline; + *now = deadline; /* Essentially make the epoll timeout = 0 */ } else if (is_polling_turn) { - *now = gpr_now(GPR_CLOCK_MONOTONIC); + *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ } - gpr_mu_lock(&pi->worker_list_mu); // Lock + gpr_mu_lock(&pi->worker_list_mu); // LOCK /* The node might have already been removed from the list by the poller that kicked this. However it is safe to call 'remove_worker_node' on an already detached node */ remove_worker_node(&worker->pi_list_link); + /* It is important to read the num_pollers again under the lock so that we + * have the latest num_pollers value that doesn't change while we are doing + * the "(num_pollers < GPRC_MAX_POLLERS_PER_ISLAND)" a a few lines below */ num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); } if (num_pollers < GRPC_MAX_POLLERS_PER_ISLAND) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); // Add a poller + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); is_lease_acquired = true; } - gpr_mu_unlock(&pi->worker_list_mu); // Unlock + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK return is_lease_acquired; } static void release_polling_lease(polling_island *pi, grpc_error **error) { gpr_mu_lock(&pi->worker_list_mu); // Lock - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); // Remove poller + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); worker_node *node = pop_front_worker_node(&pi->worker_list_head); if (node != NULL) { grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); From f3e94ecaa13433bb5310df716420da6624238f9d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 21 Apr 2017 15:48:33 -0700 Subject: [PATCH 065/177] Add TODOs to ev_epoll_linux_test --- src/core/lib/iomgr/ev_epoll_linux.c | 4 ++-- test/core/iomgr/ev_epoll_linux_test.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index a686d9e0ac3..3e13e657d35 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1472,12 +1472,12 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, gpr_timespec *now) { bool is_lease_acquired = false; - gpr_mu_lock(&pi->worker_list_mu); // LOCK + gpr_mu_lock(&pi->worker_list_mu); // LOCK long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); - gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK bool is_timeout = false; int ret; diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index f7745cddb4e..3813fb9cb07 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -419,11 +419,13 @@ static struct timespec millis_to_timespec(int millis) { return linux_ts; } +/* TODO (sreek) - Remove this test before merging. This is written just to + * understand the functionality of sigtimedwait and serves no other purpose */ void test_sigwait() { sigset_t wakeup_sig_set; sigemptyset(&wakeup_sig_set); sigaddset(&wakeup_sig_set, SIGRTMIN + 6); - int timeout_ms[] = {10, 1400}; + int timeout_ms[] = {10, 100}; for (size_t i = 0; i < GPR_ARRAY_SIZE(timeout_ms); i++) { struct timespec sigwait_timeout = millis_to_timespec(timeout_ms[i]); From 334b3548b26404bd66ddaf10c9492accc684c6f5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 17:11:29 -0700 Subject: [PATCH 066/177] Add casts --- src/core/lib/iomgr/ev_epollex_linux.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 3a0335ace0c..d12c7ffe909 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -564,7 +564,7 @@ static grpc_error *pollable_materialize(pollable *p) { if (new_epfd < 0) { return GRPC_OS_ERROR(errno, "epoll_create1"); } else { - struct epoll_event ev = {.events = EPOLLIN | EPOLLET | EPOLLEXCLUSIVE, + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), .data.ptr = &global_wakeup_fd}; if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { @@ -578,7 +578,7 @@ static grpc_error *pollable_materialize(pollable *p) { close(new_epfd); return err; } - struct epoll_event ev = {.events = EPOLLIN | EPOLLET, + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &p->wakeup}; if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, p->wakeup.read_fd, &ev) != 0) { err = GRPC_OS_ERROR(errno, "epoll_ctl"); @@ -605,7 +605,8 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { return GRPC_ERROR_NONE; } struct epoll_event ev_fd = { - .events = EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE, .data.ptr = fd}; + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT | EPOLLEXCLUSIVE), + .data.ptr = fd}; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->fd, &ev_fd) != 0) { switch (errno) { case EEXIST: /* if this fd is already in the epoll set, the workqueue fd @@ -616,8 +617,9 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { append_error(&error, GRPC_OS_ERROR(errno, "epoll_ctl"), err_desc); } } - struct epoll_event ev_wq = {.events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE, - .data.ptr = (void *)(1 + (intptr_t)fd)}; + struct epoll_event ev_wq = { + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE), + .data.ptr = (void *)(1 + (intptr_t)fd)}; if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd->workqueue_wakeup_fd.read_fd, &ev_wq) != 0) { switch (errno) { @@ -1519,7 +1521,7 @@ static bool is_epollex_available(void) { /* choose events that should cause an error on EPOLLEXCLUSIVE enabled kernels - specifically the combination of EPOLLONESHOT and EPOLLEXCLUSIVE */ - .events = EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT, + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), .data.ptr = NULL}; if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { if (errno != EINVAL) { From 3d36a5542c183d3f7cd99438dae546dbcce0f16d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 18:31:01 +0000 Subject: [PATCH 067/177] Fix compile --- test/core/iomgr/tcp_server_uv_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/iomgr/tcp_server_uv_test.c b/test/core/iomgr/tcp_server_uv_test.c index 1e039585c1e..945b84a355c 100644 --- a/test/core/iomgr/tcp_server_uv_test.c +++ b/test/core/iomgr/tcp_server_uv_test.c @@ -306,7 +306,7 @@ static void test_connect(unsigned n) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { From e0e8ed9b5bc16111057a2760b235b22bfa084483 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 18:32:11 +0000 Subject: [PATCH 068/177] Fix compile --- src/core/lib/iomgr/pollset_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 6dca37b481c..d0387d210be 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -116,7 +116,7 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } -void grpc_pollset_destroy(grpc_pollset *pollset) {} +void grpc_pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {} grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, From f62f8d37b60ebb6b5f510210320e0aca79690222 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 18:38:54 +0000 Subject: [PATCH 069/177] Revert "Better cost estimation" This reverts commit 4f98e25f8be7f77e026141adcb40bd079441be40. --- test/cpp/qps/gen_build_yaml.py | 1 - tools/run_tests/generated/tests.json | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 805b0faeece..2f035abeddc 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -66,7 +66,6 @@ def _scenario_json_string(scenario_json, is_tsan): def threads_required(scenario_json, where, is_tsan): scenario_json = mutate_scenario(scenario_json, is_tsan) if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): - if scenario_json['client_config']['client_channels'] == 1: return 1 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) return scenario_json['client_config']['outstanding_rpcs_per_channel'] * scenario_json['client_config']['client_channels'] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index d6b5cc561c8..120a84e8a4b 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -41350,7 +41350,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41452,7 +41452,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41504,7 +41504,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41929,7 +41929,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42031,7 +42031,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42083,7 +42083,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42560,7 +42560,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42714,7 +42714,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42792,7 +42792,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43438,7 +43438,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43592,7 +43592,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43670,7 +43670,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", From 956920d84ead617222fe073f2e877211dfb9ce81 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 13:09:12 -0700 Subject: [PATCH 070/177] clang-format --- src/core/lib/iomgr/ev_epollex_linux.c | 5 +++-- src/core/lib/surface/completion_queue.c | 3 ++- test/cpp/microbenchmarks/bm_pollset.cc | 11 ++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index d12c7ffe909..ee944bcfad2 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -564,8 +564,9 @@ static grpc_error *pollable_materialize(pollable *p) { if (new_epfd < 0) { return GRPC_OS_ERROR(errno, "epoll_create1"); } else { - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), - .data.ptr = &global_wakeup_fd}; + struct epoll_event ev = { + .events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), + .data.ptr = &global_wakeup_fd}; if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 7f950cdad5d..a739af2fe9a 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -98,7 +98,8 @@ static void non_polling_poller_init(grpc_pollset *pollset, gpr_mu **mu) { *mu = &npp->mu; } -static void non_polling_poller_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { +static void non_polling_poller_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { non_polling_poller *npp = (non_polling_poller *)pollset; gpr_mu_destroy(&npp->mu); } diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc index 19b3ba0a3c5..f5e8d13881f 100644 --- a/test/cpp/microbenchmarks/bm_pollset.cc +++ b/test/cpp/microbenchmarks/bm_pollset.cc @@ -157,17 +157,18 @@ static void BM_PollAddFd(benchmark::State& state) { grpc_pollset_init(ps, &mu); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_wakeup_fd wakeup_fd; - GPR_ASSERT(GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&wakeup_fd))); - grpc_fd *fd = grpc_fd_create(wakeup_fd.read_fd, "xxx"); + GPR_ASSERT( + GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&wakeup_fd))); + grpc_fd* fd = grpc_fd_create(wakeup_fd.read_fd, "xxx"); while (state.KeepRunning()) { - grpc_pollset_add_fd(&exec_ctx, ps, fd); - grpc_exec_ctx_flush(&exec_ctx); + grpc_pollset_add_fd(&exec_ctx, ps, fd); + grpc_exec_ctx_flush(&exec_ctx); } grpc_fd_orphan(&exec_ctx, fd, NULL, NULL, "xxx"); grpc_closure shutdown_ps_closure; grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps, grpc_schedule_on_exec_ctx); - gpr_mu_lock(mu); + gpr_mu_lock(mu); grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure); gpr_mu_unlock(mu); grpc_exec_ctx_finish(&exec_ctx); From c3c9bf2442664493f081460c10f98ea2c43e9496 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 13:15:05 -0700 Subject: [PATCH 071/177] Fix tracing related TSAN race --- src/core/lib/iomgr/ev_epollex_linux.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index ee944bcfad2..64afc7b804b 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -913,10 +913,9 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool write_ev = (events[i].events & EPOLLOUT) != 0; if (grpc_polling_trace) { gpr_log(GPR_DEBUG, - "PS:%p poll %p got fd %p(%d/%d): is_wq=%d cancel=%d read=%d " + "PS:%p poll %p got fd %p: is_wq=%d cancel=%d read=%d " "write=%d", - pollset, p, fd, fd->fd, fd->workqueue_wakeup_fd.read_fd, - is_workqueue, cancel, read_ev, write_ev); + pollset, p, fd, is_workqueue, cancel, read_ev, write_ev); } if (is_workqueue) { append_error(&error, From 819cd88bd7e95092f6f46fada403f86fc0dcacec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 13:18:22 -0700 Subject: [PATCH 072/177] Allow EPOLLEXCLUSIVE tests to be omitted outside of CI --- BUILD | 3 + CMakeLists.txt | 44 +++++ Makefile | 42 ++++- binding.gyp | 1 + build.yaml | 11 ++ config.m4 | 1 + gRPC-Core.podspec | 5 + grpc.gemspec | 3 + package.xml | 3 + src/core/lib/iomgr/ev_epollex_linux.c | 69 +------ .../lib/iomgr/is_epollexclusive_available.c | 116 ++++++++++++ .../lib/iomgr/is_epollexclusive_available.h | 41 +++++ src/core/lib/iomgr/sys_epoll_wrapper.h | 43 +++++ src/python/grpcio/grpc_core_dependencies.py | 1 + test/build/check_epollexclusive.c | 38 ++++ tools/doxygen/Doxyfile.c++.internal | 3 + tools/doxygen/Doxyfile.core.internal | 3 + .../generated/sources_and_headers.json | 20 +++ tools/run_tests/run_tests.py | 12 ++ vsprojects/buildtests_c.sln | 25 +++ vsprojects/grpc.sln | 25 +++ .../check_epollexclusive.vcxproj | 170 ++++++++++++++++++ .../check_epollexclusive.vcxproj.filters | 18 ++ vsprojects/vcxproj/grpc++/grpc++.vcxproj | 4 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 9 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 4 + .../grpc++_unsecure.vcxproj.filters | 9 + vsprojects/vcxproj/grpc/grpc.vcxproj | 4 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 9 + .../grpc_test_util/grpc_test_util.vcxproj | 4 + .../grpc_test_util.vcxproj.filters | 9 + .../grpc_unsecure/grpc_unsecure.vcxproj | 4 + .../grpc_unsecure.vcxproj.filters | 9 + 33 files changed, 696 insertions(+), 66 deletions(-) create mode 100644 src/core/lib/iomgr/is_epollexclusive_available.c create mode 100644 src/core/lib/iomgr/is_epollexclusive_available.h create mode 100644 src/core/lib/iomgr/sys_epoll_wrapper.h create mode 100644 test/build/check_epollexclusive.c create mode 100644 vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj create mode 100644 vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters diff --git a/BUILD b/BUILD index b3ad629272d..5944095b35d 100644 --- a/BUILD +++ b/BUILD @@ -463,6 +463,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epollex_linux.c", + "src/core/lib/iomgr/is_epollexclusive_available.c", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_posix.c", "src/core/lib/iomgr/exec_ctx.c", @@ -586,6 +587,8 @@ grpc_cc_library( "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", + "src/core/lib/iomgr/is_epollexclusive_available.h", + "src/core/lib/iomgr/sys_epoll_wrapper.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index e1513ee48f7..a3fd5b48512 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,6 +346,7 @@ add_custom_target(plugins add_custom_target(tools_c DEPENDS + check_epollexclusive gen_hpack_tables gen_legal_metadata_characters gen_percent_encoding_tables @@ -946,6 +947,7 @@ add_library(grpc src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -1272,6 +1274,7 @@ add_library(grpc_cronet src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -1583,6 +1586,7 @@ add_library(grpc_test_util src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -1840,6 +1844,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -2260,6 +2265,7 @@ add_library(grpc++ src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -2586,6 +2592,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -3351,6 +3358,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/iomgr_posix.c src/core/lib/iomgr/iomgr_uv.c src/core/lib/iomgr/iomgr_windows.c + src/core/lib/iomgr/is_epollexclusive_available.c src/core/lib/iomgr/load_file.c src/core/lib/iomgr/lockfree_event.c src/core/lib/iomgr/network_status_tracker.c @@ -5045,6 +5053,42 @@ target_link_libraries(channel_create_test ) endif (gRPC_BUILD_TESTS) + +add_executable(check_epollexclusive + test/build/check_epollexclusive.c +) + + +target_include_directories(check_epollexclusive + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(check_epollexclusive + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc + gpr +) + + +if (gRPC_INSTALL) + install(TARGETS check_epollexclusive EXPORT gRPCTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif() + if (gRPC_BUILD_TESTS) add_executable(chttp2_hpack_encoder_test diff --git a/Makefile b/Makefile index 1f356ceef3b..3540732df79 100644 --- a/Makefile +++ b/Makefile @@ -976,6 +976,7 @@ census_context_test: $(BINDIR)/$(CONFIG)/census_context_test census_resource_test: $(BINDIR)/$(CONFIG)/census_resource_test census_trace_context_test: $(BINDIR)/$(CONFIG)/census_trace_context_test channel_create_test: $(BINDIR)/$(CONFIG)/channel_create_test +check_epollexclusive: $(BINDIR)/$(CONFIG)/check_epollexclusive chttp2_hpack_encoder_test: $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test @@ -2133,7 +2134,7 @@ test_python: static_c tools: tools_c tools_cxx -tools_c: privatelibs_c $(BINDIR)/$(CONFIG)/gen_hpack_tables $(BINDIR)/$(CONFIG)/gen_legal_metadata_characters $(BINDIR)/$(CONFIG)/gen_percent_encoding_tables $(BINDIR)/$(CONFIG)/grpc_create_jwt $(BINDIR)/$(CONFIG)/grpc_print_google_default_creds_token $(BINDIR)/$(CONFIG)/grpc_verify_jwt +tools_c: privatelibs_c $(BINDIR)/$(CONFIG)/check_epollexclusive $(BINDIR)/$(CONFIG)/gen_hpack_tables $(BINDIR)/$(CONFIG)/gen_legal_metadata_characters $(BINDIR)/$(CONFIG)/gen_percent_encoding_tables $(BINDIR)/$(CONFIG)/grpc_create_jwt $(BINDIR)/$(CONFIG)/grpc_print_google_default_creds_token $(BINDIR)/$(CONFIG)/grpc_verify_jwt tools_cxx: privatelibs_cxx @@ -2920,6 +2921,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -3244,6 +3246,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -3554,6 +3557,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -3783,6 +3787,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -4180,6 +4185,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -4514,6 +4520,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -5276,6 +5283,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ @@ -8991,6 +8999,38 @@ endif endif +CHECK_EPOLLEXCLUSIVE_SRC = \ + test/build/check_epollexclusive.c \ + +CHECK_EPOLLEXCLUSIVE_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CHECK_EPOLLEXCLUSIVE_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/check_epollexclusive: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/check_epollexclusive: $(CHECK_EPOLLEXCLUSIVE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(CHECK_EPOLLEXCLUSIVE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/check_epollexclusive + +endif + +$(OBJDIR)/$(CONFIG)/test/build/check_epollexclusive.o: $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_check_epollexclusive: $(CHECK_EPOLLEXCLUSIVE_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CHECK_EPOLLEXCLUSIVE_OBJS:.o=.dep) +endif +endif + + CHTTP2_HPACK_ENCODER_TEST_SRC = \ test/core/transport/chttp2/hpack_encoder_test.c \ diff --git a/binding.gyp b/binding.gyp index 82242c057c1..d39665272cb 100644 --- a/binding.gyp +++ b/binding.gyp @@ -683,6 +683,7 @@ 'src/core/lib/iomgr/iomgr_posix.c', 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', + 'src/core/lib/iomgr/is_epollexclusive_available.c', 'src/core/lib/iomgr/load_file.c', 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', diff --git a/build.yaml b/build.yaml index 80ad5498f16..856104dbe2f 100644 --- a/build.yaml +++ b/build.yaml @@ -203,6 +203,7 @@ filegroups: - src/core/lib/iomgr/iomgr.h - src/core/lib/iomgr/iomgr_internal.h - src/core/lib/iomgr/iomgr_posix.h + - src/core/lib/iomgr/is_epollexclusive_available.h - src/core/lib/iomgr/load_file.h - src/core/lib/iomgr/lockfree_event.h - src/core/lib/iomgr/network_status_tracker.h @@ -224,6 +225,7 @@ filegroups: - src/core/lib/iomgr/socket_utils.h - src/core/lib/iomgr/socket_utils_posix.h - src/core/lib/iomgr/socket_windows.h + - src/core/lib/iomgr/sys_epoll_wrapper.h - src/core/lib/iomgr/tcp_client.h - src/core/lib/iomgr/tcp_client_posix.h - src/core/lib/iomgr/tcp_posix.h @@ -312,6 +314,7 @@ filegroups: - src/core/lib/iomgr/iomgr_posix.c - src/core/lib/iomgr/iomgr_uv.c - src/core/lib/iomgr/iomgr_windows.c + - src/core/lib/iomgr/is_epollexclusive_available.c - src/core/lib/iomgr/load_file.c - src/core/lib/iomgr/lockfree_event.c - src/core/lib/iomgr/network_status_tracker.c @@ -1658,6 +1661,14 @@ targets: - grpc - gpr_test_util - gpr +- name: check_epollexclusive + build: tool + language: c + src: + - test/build/check_epollexclusive.c + deps: + - grpc + - gpr - name: chttp2_hpack_encoder_test build: test language: c diff --git a/config.m4 b/config.m4 index 73c0e01a38f..8d07fa931ee 100644 --- a/config.m4 +++ b/config.m4 @@ -119,6 +119,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ + src/core/lib/iomgr/is_epollexclusive_available.c \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/lockfree_event.c \ src/core/lib/iomgr/network_status_tracker.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c5ff33ef870..3a1741cc851 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -285,6 +285,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_internal.h', 'src/core/lib/iomgr/iomgr_posix.h', + 'src/core/lib/iomgr/is_epollexclusive_available.h', 'src/core/lib/iomgr/load_file.h', 'src/core/lib/iomgr/lockfree_event.h', 'src/core/lib/iomgr/network_status_tracker.h', @@ -306,6 +307,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils.h', 'src/core/lib/iomgr/socket_utils_posix.h', 'src/core/lib/iomgr/socket_windows.h', + 'src/core/lib/iomgr/sys_epoll_wrapper.h', 'src/core/lib/iomgr/tcp_client.h', 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', @@ -492,6 +494,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr_posix.c', 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', + 'src/core/lib/iomgr/is_epollexclusive_available.c', 'src/core/lib/iomgr/load_file.c', 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', @@ -742,6 +745,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/iomgr.h', 'src/core/lib/iomgr/iomgr_internal.h', 'src/core/lib/iomgr/iomgr_posix.h', + 'src/core/lib/iomgr/is_epollexclusive_available.h', 'src/core/lib/iomgr/load_file.h', 'src/core/lib/iomgr/lockfree_event.h', 'src/core/lib/iomgr/network_status_tracker.h', @@ -763,6 +767,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/socket_utils.h', 'src/core/lib/iomgr/socket_utils_posix.h', 'src/core/lib/iomgr/socket_windows.h', + 'src/core/lib/iomgr/sys_epoll_wrapper.h', 'src/core/lib/iomgr/tcp_client.h', 'src/core/lib/iomgr/tcp_client_posix.h', 'src/core/lib/iomgr/tcp_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index 33822cc649a..b810d02d73d 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -201,6 +201,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/iomgr.h ) s.files += %w( src/core/lib/iomgr/iomgr_internal.h ) s.files += %w( src/core/lib/iomgr/iomgr_posix.h ) + s.files += %w( src/core/lib/iomgr/is_epollexclusive_available.h ) s.files += %w( src/core/lib/iomgr/load_file.h ) s.files += %w( src/core/lib/iomgr/lockfree_event.h ) s.files += %w( src/core/lib/iomgr/network_status_tracker.h ) @@ -222,6 +223,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/socket_utils.h ) s.files += %w( src/core/lib/iomgr/socket_utils_posix.h ) s.files += %w( src/core/lib/iomgr/socket_windows.h ) + s.files += %w( src/core/lib/iomgr/sys_epoll_wrapper.h ) s.files += %w( src/core/lib/iomgr/tcp_client.h ) s.files += %w( src/core/lib/iomgr/tcp_client_posix.h ) s.files += %w( src/core/lib/iomgr/tcp_posix.h ) @@ -408,6 +410,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/iomgr_posix.c ) s.files += %w( src/core/lib/iomgr/iomgr_uv.c ) s.files += %w( src/core/lib/iomgr/iomgr_windows.c ) + s.files += %w( src/core/lib/iomgr/is_epollexclusive_available.c ) s.files += %w( src/core/lib/iomgr/load_file.c ) s.files += %w( src/core/lib/iomgr/lockfree_event.c ) s.files += %w( src/core/lib/iomgr/network_status_tracker.c ) diff --git a/package.xml b/package.xml index 3f438a51cef..9c52f5cbe8e 100644 --- a/package.xml +++ b/package.xml @@ -210,6 +210,7 @@ + @@ -231,6 +232,7 @@ + @@ -417,6 +419,7 @@ + diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 64afc7b804b..b7c3e2e959f 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -43,7 +43,6 @@ #include #include #include -#include #include #include @@ -55,7 +54,9 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/is_epollexclusive_available.h" #include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/sys_epoll_wrapper.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" @@ -63,10 +64,6 @@ #include "src/core/lib/support/block_annotate.h" #include "src/core/lib/support/spinlock.h" -#ifndef EPOLLEXCLUSIVE -#define EPOLLEXCLUSIVE (1 << 28) -#endif - /* TODO: sreek: Right now, this wakes up all pollers. In future we should make * sure to wake up one polling thread (which can wake up other threads if * needed) */ @@ -1496,70 +1493,12 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -/* It is possible that GLIBC has epoll but the underlying kernel doesn't. - * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epollex_available(void) { - static bool logged_why_not = false; - - int fd = epoll_create1(EPOLL_CLOEXEC); - if (fd < 0) { - if (!logged_why_not) { - gpr_log(GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epollex polling " - "engine.", - fd); - logged_why_not = true; - } - return false; - } - grpc_wakeup_fd wakeup; - if (!GRPC_LOG_IF_ERROR("check_wakeupfd_for_epollex", - grpc_wakeup_fd_init(&wakeup))) { - return false; - } - struct epoll_event ev = { - /* choose events that should cause an error on - EPOLLEXCLUSIVE enabled kernels - specifically the combination of - EPOLLONESHOT and EPOLLEXCLUSIVE */ - .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), - .data.ptr = NULL}; - if (epoll_ctl(fd, EPOLL_CTL_ADD, wakeup.read_fd, &ev) != 0) { - if (errno != EINVAL) { - if (!logged_why_not) { - gpr_log( - GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " - "%d. Not using epollex polling engine.", - errno); - logged_why_not = true; - } - close(fd); - grpc_wakeup_fd_destroy(&wakeup); - return false; - } - } else { - if (!logged_why_not) { - gpr_log(GPR_ERROR, - "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " - "evidence of no EPOLLEXCLUSIVE support. Not using " - "epollex polling engine."); - logged_why_not = true; - } - close(fd); - grpc_wakeup_fd_destroy(&wakeup); - return false; - } - grpc_wakeup_fd_destroy(&wakeup); - close(fd); - return true; -} - const grpc_event_engine_vtable *grpc_init_epollex_linux(void) { if (!grpc_has_wakeup_fd()) { return NULL; } - if (!is_epollex_available()) { + if (!grpc_is_epollexclusive_available()) { return NULL; } diff --git a/src/core/lib/iomgr/is_epollexclusive_available.c b/src/core/lib/iomgr/is_epollexclusive_available.c new file mode 100644 index 00000000000..34fcf313e5e --- /dev/null +++ b/src/core/lib/iomgr/is_epollexclusive_available.c @@ -0,0 +1,116 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +#include "src/core/lib/iomgr/is_epollexclusive_available.h" + +#ifdef GRPC_LINUX_EPOLL + +#include + +#include +#include +#include + +#include "src/core/lib/iomgr/sys_epoll_wrapper.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +bool grpc_is_epollexclusive_available(void) { + static bool logged_why_not = false; + + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } + return false; + } + int evfd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (evfd < 0) { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "eventfd failed with error: %d. Not using epollex polling " + "engine.", + fd); + logged_why_not = true; + } + close(fd); + return false; + } + struct epoll_event ev = { + /* choose events that should cause an error on + EPOLLEXCLUSIVE enabled kernels - specifically the combination of + EPOLLONESHOT and EPOLLEXCLUSIVE */ + .events = (uint32_t)(EPOLLET | EPOLLIN | EPOLLEXCLUSIVE | EPOLLONESHOT), + .data.ptr = NULL}; + if (epoll_ctl(fd, EPOLL_CTL_ADD, evfd, &ev) != 0) { + if (errno != EINVAL) { + if (!logged_why_not) { + gpr_log( + GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT failed with error: " + "%d. Not using epollex polling engine.", + errno); + logged_why_not = true; + } + close(fd); + close(evfd); + return false; + } + } else { + if (!logged_why_not) { + gpr_log(GPR_ERROR, + "epoll_ctl with EPOLLEXCLUSIVE | EPOLLONESHOT succeeded. This is " + "evidence of no EPOLLEXCLUSIVE support. Not using " + "epollex polling engine."); + logged_why_not = true; + } + close(fd); + close(evfd); + return false; + } + close(evfd); + close(fd); + return true; +} + +#else + +bool grpc_is_epollexclusive_available(void) { return false; } + +#endif diff --git a/src/core/lib/iomgr/is_epollexclusive_available.h b/src/core/lib/iomgr/is_epollexclusive_available.h new file mode 100644 index 00000000000..b65b819e740 --- /dev/null +++ b/src/core/lib/iomgr/is_epollexclusive_available.h @@ -0,0 +1,41 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H +#define GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H + +#include + +bool grpc_is_epollexclusive_available(void); + +#endif /* GRPC_CORE_LIB_IOMGR_IS_EPOLLEXCLUSIVE_AVAILABLE_H */ diff --git a/src/core/lib/iomgr/sys_epoll_wrapper.h b/src/core/lib/iomgr/sys_epoll_wrapper.h new file mode 100644 index 00000000000..2f08423193d --- /dev/null +++ b/src/core/lib/iomgr/sys_epoll_wrapper.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H +#define GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H + +#include + +#ifndef EPOLLEXCLUSIVE +#define EPOLLEXCLUSIVE (1 << 28) +#endif + +#endif /* GRPC_CORE_LIB_IOMGR_SYS_EPOLL_WRAPPER_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d3af7cace3a..ced072bf59b 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -108,6 +108,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/iomgr_posix.c', 'src/core/lib/iomgr/iomgr_uv.c', 'src/core/lib/iomgr/iomgr_windows.c', + 'src/core/lib/iomgr/is_epollexclusive_available.c', 'src/core/lib/iomgr/load_file.c', 'src/core/lib/iomgr/lockfree_event.c', 'src/core/lib/iomgr/network_status_tracker.c', diff --git a/test/build/check_epollexclusive.c b/test/build/check_epollexclusive.c new file mode 100644 index 00000000000..fb512c3ae1a --- /dev/null +++ b/test/build/check_epollexclusive.c @@ -0,0 +1,38 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/is_epollexclusive_available.h" + +int main(int argc, char **argv) { + return grpc_is_epollexclusive_available() ? 0 : 1; +} diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index f0768464d82..4cf5bef7471 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -960,6 +960,8 @@ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_posix.h \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ +src/core/lib/iomgr/is_epollexclusive_available.c \ +src/core/lib/iomgr/is_epollexclusive_available.h \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/load_file.h \ src/core/lib/iomgr/lockfree_event.c \ @@ -1002,6 +1004,7 @@ src/core/lib/iomgr/socket_utils_uv.c \ src/core/lib/iomgr/socket_utils_windows.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/socket_windows.h \ +src/core/lib/iomgr/sys_epoll_wrapper.h \ src/core/lib/iomgr/tcp_client.h \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_posix.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 02af1d69cbc..d43e4bcbc51 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1095,6 +1095,8 @@ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_posix.h \ src/core/lib/iomgr/iomgr_uv.c \ src/core/lib/iomgr/iomgr_windows.c \ +src/core/lib/iomgr/is_epollexclusive_available.c \ +src/core/lib/iomgr/is_epollexclusive_available.h \ src/core/lib/iomgr/load_file.c \ src/core/lib/iomgr/load_file.h \ src/core/lib/iomgr/lockfree_event.c \ @@ -1137,6 +1139,7 @@ src/core/lib/iomgr/socket_utils_uv.c \ src/core/lib/iomgr/socket_utils_windows.c \ src/core/lib/iomgr/socket_windows.c \ src/core/lib/iomgr/socket_windows.h \ +src/core/lib/iomgr/sys_epoll_wrapper.h \ src/core/lib/iomgr/tcp_client.h \ src/core/lib/iomgr/tcp_client_posix.c \ src/core/lib/iomgr/tcp_client_posix.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6ef1c13b102..36305a003aa 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -232,6 +232,21 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "grpc" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "check_epollexclusive", + "src": [ + "test/build/check_epollexclusive.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -7711,6 +7726,7 @@ "src/core/lib/iomgr/iomgr.h", "src/core/lib/iomgr/iomgr_internal.h", "src/core/lib/iomgr/iomgr_posix.h", + "src/core/lib/iomgr/is_epollexclusive_available.h", "src/core/lib/iomgr/load_file.h", "src/core/lib/iomgr/lockfree_event.h", "src/core/lib/iomgr/network_status_tracker.h", @@ -7732,6 +7748,7 @@ "src/core/lib/iomgr/socket_utils.h", "src/core/lib/iomgr/socket_utils_posix.h", "src/core/lib/iomgr/socket_windows.h", + "src/core/lib/iomgr/sys_epoll_wrapper.h", "src/core/lib/iomgr/tcp_client.h", "src/core/lib/iomgr/tcp_client_posix.h", "src/core/lib/iomgr/tcp_posix.h", @@ -7864,6 +7881,8 @@ "src/core/lib/iomgr/iomgr_posix.h", "src/core/lib/iomgr/iomgr_uv.c", "src/core/lib/iomgr/iomgr_windows.c", + "src/core/lib/iomgr/is_epollexclusive_available.c", + "src/core/lib/iomgr/is_epollexclusive_available.h", "src/core/lib/iomgr/load_file.c", "src/core/lib/iomgr/load_file.h", "src/core/lib/iomgr/lockfree_event.c", @@ -7906,6 +7925,7 @@ "src/core/lib/iomgr/socket_utils_windows.c", "src/core/lib/iomgr/socket_windows.c", "src/core/lib/iomgr/socket_windows.h", + "src/core/lib/iomgr/sys_epoll_wrapper.h", "src/core/lib/iomgr/tcp_client.h", "src/core/lib/iomgr/tcp_client_posix.c", "src/core/lib/iomgr/tcp_client_posix.h", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d2e3506fe5a..3698d1b8618 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1432,6 +1432,14 @@ class BuildAndRunError(object): POST_TEST = object() +def _has_epollexclusive(): + try: + subprocess.check_call('bins/%s/check_epollexclusive' % args.config) + return True + except subprocess.CalledProcessError, e: + return False + + # returns a list of things that failed (or an empty list on success) def _build_and_run( check_cancelled, newline_on_success, xml_report=None, build_only=False): @@ -1449,6 +1457,10 @@ def _build_and_run( suite_name=args.report_suite_name) return [] + if not args.travis and not _has_epollexclusive() and 'epollex' in _POLLING_STRATEGIES[platform_string()]: + print('\n\nOmitting EPOLLEXCLUSIVE tests\n\n') + _POLLING_STRATEGIES[platform_string()].remove('epollex') + # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/python_utils/antagonist.py']) for _ in range(0, args.antagonists)] diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 539f474f9ec..2e8ccf812bd 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -162,6 +162,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel_create_test", "vcxp {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "check_epollexclusive", "vcxproj\.\check_epollexclusive\check_epollexclusive.vcxproj", "{03306445-5BA0-289C-02AB-513DE6C52124}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "chttp2_hpack_encoder_test", "vcxproj\test\chttp2_hpack_encoder_test\chttp2_hpack_encoder_test.vcxproj", "{19F92966-3B0E-4FF8-CD7C-435D353E079E}" ProjectSection(myProperties) = preProject lib = "False" @@ -1914,6 +1923,22 @@ Global {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release-DLL|Win32.Build.0 = Release|Win32 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release-DLL|x64.ActiveCfg = Release|x64 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release-DLL|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.Build.0 = Release|x64 {19F92966-3B0E-4FF8-CD7C-435D353E079E}.Debug|Win32.ActiveCfg = Debug|Win32 {19F92966-3B0E-4FF8-CD7C-435D353E079E}.Debug|x64.ActiveCfg = Debug|x64 {19F92966-3B0E-4FF8-CD7C-435D353E079E}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/grpc.sln b/vsprojects/grpc.sln index 97378e0a0f8..307ae4b5990 100644 --- a/vsprojects/grpc.sln +++ b/vsprojects/grpc.sln @@ -8,6 +8,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ares", "vcxproj\.\ares\ares lib = "True" EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "check_epollexclusive", "vcxproj\.\check_epollexclusive\check_epollexclusive.vcxproj", "{03306445-5BA0-289C-02AB-513DE6C52124}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "gen_hpack_tables", "vcxproj\.\gen_hpack_tables\gen_hpack_tables.vcxproj", "{FCDEA4C7-7F26-05DB-D08F-A08F499026E6}" ProjectSection(myProperties) = preProject lib = "False" @@ -190,6 +199,22 @@ Global {1769D06D-F18C-B4C2-B019-31D7F83F3C9A}.Release-DLL|Win32.Build.0 = Release|Win32 {1769D06D-F18C-B4C2-B019-31D7F83F3C9A}.Release-DLL|x64.ActiveCfg = Release|x64 {1769D06D-F18C-B4C2-B019-31D7F83F3C9A}.Release-DLL|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release|x64.Build.0 = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Debug-DLL|x64.Build.0 = Debug|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|Win32.Build.0 = Release|Win32 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.ActiveCfg = Release|x64 + {03306445-5BA0-289C-02AB-513DE6C52124}.Release-DLL|x64.Build.0 = Release|x64 {FCDEA4C7-7F26-05DB-D08F-A08F499026E6}.Debug|Win32.ActiveCfg = Debug|Win32 {FCDEA4C7-7F26-05DB-D08F-A08F499026E6}.Debug|x64.ActiveCfg = Debug|x64 {FCDEA4C7-7F26-05DB-D08F-A08F499026E6}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj new file mode 100644 index 00000000000..36e6fee0db8 --- /dev/null +++ b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj @@ -0,0 +1,170 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {03306445-5BA0-289C-02AB-513DE6C52124} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + StaticLibrary + true + Unicode + + + StaticLibrary + false + true + Unicode + + + + + + + + + + + + check_epollexclusive + + + check_epollexclusive + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + diff --git a/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters new file mode 100644 index 00000000000..5572541d1da --- /dev/null +++ b/vsprojects/vcxproj/check_epollexclusive/check_epollexclusive.vcxproj.filters @@ -0,0 +1,18 @@ + + + + + test\build + + + + + + {98195cc4-af1b-3ef2-80a8-86b96fa1c488} + + + {c32fe5ee-bf78-58e4-fa70-b9294e48a136} + + + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index fa4610974ae..3648bd67a24 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -406,6 +406,7 @@ + @@ -427,6 +428,7 @@ + @@ -633,6 +635,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index e94aee357e7..2703372d1d8 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -217,6 +217,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -941,6 +944,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -1004,6 +1010,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 5513816d774..b6bfe437be3 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -400,6 +400,7 @@ + @@ -421,6 +422,7 @@ + @@ -617,6 +619,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 646e35bbe8f..6a86c11a3e0 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -202,6 +202,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -908,6 +911,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -971,6 +977,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index d6881a98c0b..7e864c10b2b 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -330,6 +330,7 @@ + @@ -351,6 +352,7 @@ + @@ -571,6 +573,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 2a694cd4f3d..ba17b67ed3a 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -97,6 +97,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -899,6 +902,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -962,6 +968,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 1a3120c7988..55d53eec476 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -225,6 +225,7 @@ + @@ -246,6 +247,7 @@ + @@ -406,6 +408,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index d750cf78d99..0a12bc9ee12 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -154,6 +154,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -656,6 +659,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -719,6 +725,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0716de4a9c4..bfa01f1a3c0 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -320,6 +320,7 @@ + @@ -341,6 +342,7 @@ + @@ -539,6 +541,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index ebbd03b5fb4..e23e2cf0911 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -100,6 +100,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -812,6 +815,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -875,6 +881,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From 7f88c484113ec5174209ac124ff7b0ce32e72f49 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 25 Apr 2017 16:44:17 -0700 Subject: [PATCH 073/177] Fix portability test --- test/core/iomgr/tcp_client_uv_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c index 92fc3934226..3a8458df861 100644 --- a/test/core/iomgr/tcp_client_uv_test.c +++ b/test/core/iomgr/tcp_client_uv_test.c @@ -194,7 +194,7 @@ void test_fails(void) { static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, grpc_error *error) { - grpc_pollset_destroy(p); + grpc_pollset_destroy(exec_ctx, p); } int main(int argc, char **argv) { From 22b8bb1e0824eb21b3f244e543589639a85fb36d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 26 Apr 2017 20:08:30 -0700 Subject: [PATCH 074/177] Make max-pollers configurable via GRPC_MAX_POLLERS_PER_PI env variable --- src/core/lib/iomgr/ev_epoll_linux.c | 53 +++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 3e13e657d35..3a5f48cb8f0 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -62,6 +63,7 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/env.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ static int grpc_polling_trace = 0; /* Disabled by default */ @@ -73,6 +75,10 @@ static int grpc_polling_trace = 0; /* Disabled by default */ /* Uncomment the following to enable extra checks on poll_object operations */ /* #define PO_DEBUG */ +/* The maximum number of polling threads per polling island. By default no + limit */ +static int g_max_pollers_per_pi = INT_MAX; + static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; @@ -97,9 +103,6 @@ void grpc_use_signal(int signum) { } } -/* The maximum number of polling threads per polling island */ -#define GRPC_MAX_POLLERS_PER_ISLAND 1 - struct polling_island; typedef enum { @@ -1466,7 +1469,7 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, return false; } -/* NOTE: May modify 'now' */ +/* NOTE: This function may modify 'now' */ static bool acquire_polling_lease(grpc_pollset_worker *worker, polling_island *pi, gpr_timespec deadline, gpr_timespec *now) { @@ -1475,7 +1478,7 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, gpr_mu_lock(&pi->worker_list_mu); // LOCK long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - if (num_pollers >= GRPC_MAX_POLLERS_PER_ISLAND) { + if (num_pollers >= g_max_pollers_per_pi) { push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK @@ -1493,20 +1496,21 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, if (errno == EAGAIN) { is_timeout = true; } else { - /* TODO: sreek This should not happen. If we see these log messages, it - * means we are most likely something incorrect in the setup needed for - * sigwaitinfo/sigtimedwait */ - gpr_log(GPR_ERROR, "Failed with retcode: %d (timeout_ms: %d)", errno, + /* NOTE: This should not happen. If we see these log messages, it means + we are most likely doing something incorrect in the setup * needed + for sigwaitinfo/sigtimedwait */ + gpr_log(GPR_ERROR, + "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, timeout_ms); } } /* Did the worker come out of sigtimedwait due to a thread that just - * exited epoll and kicking it (in release_polling_lease function). */ + exited epoll and kicking it (in release_polling_lease function). */ bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); /* Did the worker come out of sigtimedwait due to a thread alerting it that - * some completion event was (likely) available in the completion queue */ + some completion event was (likely) available in the completion queue */ bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); if (is_kicked || is_timeout) { @@ -1522,11 +1526,11 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, remove_worker_node(&worker->pi_list_link); /* It is important to read the num_pollers again under the lock so that we * have the latest num_pollers value that doesn't change while we are doing - * the "(num_pollers < GPRC_MAX_POLLERS_PER_ISLAND)" a a few lines below */ + * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); } - if (num_pollers < GRPC_MAX_POLLERS_PER_ISLAND) { + if (num_pollers < g_max_pollers_per_pi) { gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); is_lease_acquired = true; } @@ -1536,7 +1540,7 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, } static void release_polling_lease(polling_island *pi, grpc_error **error) { - gpr_mu_lock(&pi->worker_list_mu); // Lock + gpr_mu_lock(&pi->worker_list_mu); gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); worker_node *node = pop_front_worker_node(&pi->worker_list_head); @@ -1554,6 +1558,8 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, grpc_pollset_worker *worker, gpr_timespec now, gpr_timespec deadline, sigset_t *sig_mask, grpc_error **error) { + /* Only g_max_pollers_per_pi threads can be doing polling in parallel. + If we cannot get a lease, we cannot continue to do epoll_pwait() */ if (!acquire_polling_lease(worker, pi, deadline, &now)) { return; } @@ -1563,6 +1569,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, char *err_msg; const char *err_desc = "pollset_work_and_unlock"; + /* timeout_ms is the time between 'now' and 'deadline' */ int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); GRPC_SCHEDULING_START_BLOCKING_REGION; @@ -1570,6 +1577,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); GRPC_SCHEDULING_END_BLOCKING_REGION; + /* Give back the lease right away so that some other thread can enter */ release_polling_lease(pi, error); if (ep_rv < 0) { @@ -2116,6 +2124,21 @@ static bool is_epoll_available() { return true; } +/* This is mainly for testing purposes. Checks to see if environment variable + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to the */ +static void set_max_pollers_per_island() { + char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); + if (s) { + int max_pollers = (int)strtol(s, NULL, 10); + if (max_pollers > 0) { + g_max_pollers_per_pi = max_pollers; + } + } + + gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", + g_max_pollers_per_pi); +} + const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { @@ -2134,6 +2157,8 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { grpc_use_signal(SIGRTMIN + 6); } + set_max_pollers_per_island(); + fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { From 93478976b7bc72e1cf44c843fa5505b0e5caf97d Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 26 Apr 2017 20:39:03 -0700 Subject: [PATCH 075/177] Cleanup ev_epoll_linux_test and add some log statements in ev_epoll_linux.c --- src/core/lib/iomgr/ev_epoll_linux.c | 11 +++--- test/core/iomgr/ev_epoll_linux_test.c | 48 ++------------------------- 2 files changed, 9 insertions(+), 50 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 3a5f48cb8f0..1234828063b 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -2125,14 +2125,17 @@ static bool is_epoll_available() { } /* This is mainly for testing purposes. Checks to see if environment variable - * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to the */ + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to + * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ static void set_max_pollers_per_island() { char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); if (s) { - int max_pollers = (int)strtol(s, NULL, 10); - if (max_pollers > 0) { - g_max_pollers_per_pi = max_pollers; + g_max_pollers_per_pi = (int)strtol(s, NULL, 10); + if (g_max_pollers_per_pi < 0) { + g_max_pollers_per_pi = INT_MAX; } + } else { + g_max_pollers_per_pi = INT_MAX; } gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epoll_linux_test.c index 3813fb9cb07..0856023b14f 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epoll_linux_test.c @@ -38,10 +38,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include -#include -#include #include -#include #include #include @@ -330,7 +327,7 @@ static __thread int thread_wakeups = 0; static void test_threading_loop(void *arg) { threading_shared *shared = arg; - while (thread_wakeups < 20000) { + while (thread_wakeups < 1000000) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_pollset_worker *worker; gpr_mu_lock(shared->mu); @@ -363,7 +360,7 @@ static void test_threading(void) { shared.pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(shared.pollset, &shared.mu); - gpr_thd_id thds[20]; + gpr_thd_id thds[10]; for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) { gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); @@ -402,46 +399,6 @@ static void test_threading(void) { gpr_free(shared.pollset); } -/* Convert milliseconds into 'struct timespec' struct. millis == -1 is - * * considered as an infinity-time in future */ -static struct timespec millis_to_timespec(int millis) { - struct timespec linux_ts; - gpr_timespec gpr_ts; - - if (millis == -1) { - gpr_ts = gpr_inf_future(GPR_TIMESPAN); - } else { - gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); - } - - linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; - linux_ts.tv_nsec = gpr_ts.tv_nsec; - return linux_ts; -} - -/* TODO (sreek) - Remove this test before merging. This is written just to - * understand the functionality of sigtimedwait and serves no other purpose */ -void test_sigwait() { - sigset_t wakeup_sig_set; - sigemptyset(&wakeup_sig_set); - sigaddset(&wakeup_sig_set, SIGRTMIN + 6); - int timeout_ms[] = {10, 100}; - - for (size_t i = 0; i < GPR_ARRAY_SIZE(timeout_ms); i++) { - struct timespec sigwait_timeout = millis_to_timespec(timeout_ms[i]); - gpr_log(GPR_ERROR, "sigwait_timeout: %ld, %ld", sigwait_timeout.tv_sec, - sigwait_timeout.tv_nsec); - - gpr_log(GPR_ERROR, "Waiting for %d ms...", timeout_ms[i]); - gpr_timespec bef = gpr_now(GPR_CLOCK_REALTIME); - sigtimedwait(&wakeup_sig_set, NULL, &sigwait_timeout); - gpr_timespec af = gpr_now(GPR_CLOCK_REALTIME); - - gpr_log(GPR_ERROR, "Bef: %ld, %d", bef.tv_sec, bef.tv_nsec); - gpr_log(GPR_ERROR, "Aft: %ld, %d", af.tv_sec, af.tv_nsec); - } -} - int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); @@ -452,7 +409,6 @@ int main(int argc, char **argv) { test_add_fd_to_pollset(); test_pollset_queue_merge_items(); test_threading(); - test_sigwait(); } else { gpr_log(GPR_INFO, "Skipping the test. The test is only relevant for 'epoll' " From c67cc999e35061873b4ba8bd72ebfb4ff8b7c0b6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 10:15:51 -0700 Subject: [PATCH 076/177] Initial fork for singleton epoll poller --- CMakeLists.txt | 21 +- Makefile | 21 +- binding.gyp | 3 +- build.yaml | 6 +- config.m4 | 3 +- gRPC-Core.podspec | 9 +- grpc.gemspec | 6 +- package.xml | 6 +- src/core/lib/iomgr/ev_epoll1_linux.c | 579 ++++++++++++++++++ src/core/lib/iomgr/ev_epoll1_linux.h | 50 ++ .../{ev_epoll_linux.c => ev_epollsig_linux.c} | 6 +- .../{ev_epoll_linux.h => ev_epollsig_linux.h} | 8 +- src/core/lib/iomgr/ev_posix.c | 6 +- src/python/grpcio/grpc_core_dependencies.py | 3 +- tools/doxygen/Doxyfile.c++.internal | 6 +- tools/doxygen/Doxyfile.core.internal | 6 +- .../generated/sources_and_headers.json | 9 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 7 +- .../vcxproj/grpc++/grpc++.vcxproj.filters | 10 +- .../grpc++_unsecure/grpc++_unsecure.vcxproj | 7 +- .../grpc++_unsecure.vcxproj.filters | 10 +- vsprojects/vcxproj/grpc/grpc.vcxproj | 7 +- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 10 +- .../grpc_test_util/grpc_test_util.vcxproj | 7 +- .../grpc_test_util.vcxproj.filters | 10 +- .../grpc_unsecure/grpc_unsecure.vcxproj | 7 +- .../grpc_unsecure.vcxproj.filters | 10 +- 27 files changed, 771 insertions(+), 62 deletions(-) create mode 100644 src/core/lib/iomgr/ev_epoll1_linux.c create mode 100644 src/core/lib/iomgr/ev_epoll1_linux.h rename src/core/lib/iomgr/{ev_epoll_linux.c => ev_epollsig_linux.c} (99%) rename src/core/lib/iomgr/{ev_epoll_linux.h => ev_epollsig_linux.h} (89%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fd3b40cfaa..2debe0422e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -936,7 +936,8 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1261,7 +1262,8 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1571,7 +1573,8 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1827,7 +1830,8 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2246,7 +2250,8 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2571,7 +2576,8 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -3335,7 +3341,8 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll1_linux.c + src/core/lib/iomgr/ev_epollsig_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c diff --git a/Makefile b/Makefile index 04e681480b0..b5d8b5b2549 100644 --- a/Makefile +++ b/Makefile @@ -2916,7 +2916,8 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3239,7 +3240,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3548,7 +3550,8 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3776,7 +3779,8 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4172,7 +4176,8 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4505,7 +4510,8 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -5266,7 +5272,8 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/binding.gyp b/binding.gyp index 7974a7eb043..70538f9b431 100644 --- a/binding.gyp +++ b/binding.gyp @@ -672,7 +672,8 @@ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll1_linux.c', + 'src/core/lib/iomgr/ev_epollsig_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/build.yaml b/build.yaml index f5ada4f34f3..89b356c8f2c 100644 --- a/build.yaml +++ b/build.yaml @@ -197,7 +197,8 @@ filegroups: - src/core/lib/iomgr/endpoint_pair.h - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h - - src/core/lib/iomgr/ev_epoll_linux.h + - src/core/lib/iomgr/ev_epoll1_linux.h + - src/core/lib/iomgr/ev_epollsig_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -304,7 +305,8 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_uv.c - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - - src/core/lib/iomgr/ev_epoll_linux.c + - src/core/lib/iomgr/ev_epoll1_linux.c + - src/core/lib/iomgr/ev_epollsig_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c - src/core/lib/iomgr/exec_ctx.c diff --git a/config.m4 b/config.m4 index 98e48e54542..ddd260816ce 100644 --- a/config.m4 +++ b/config.m4 @@ -108,7 +108,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll1_linux.c \ + src/core/lib/iomgr/ev_epollsig_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index a81d37cd636..f970a8ec01a 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -279,7 +279,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', - 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll1_linux.h', + 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -484,7 +485,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll1_linux.c', + 'src/core/lib/iomgr/ev_epollsig_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', @@ -738,7 +740,8 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', - 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll1_linux.h', + 'src/core/lib/iomgr/ev_epollsig_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/grpc.gemspec b/grpc.gemspec index 677839d495e..c23bcd7083f 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -195,7 +195,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair.h ) s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) - s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -400,7 +401,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_uv.c ) s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) - s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) diff --git a/package.xml b/package.xml index 771e34cffc4..2081b0c3b7a 100644 --- a/package.xml +++ b/package.xml @@ -204,7 +204,8 @@ - + + @@ -409,7 +410,8 @@ - + + diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c new file mode 100644 index 00000000000..3c7a16b5ed4 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -0,0 +1,579 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; +static int g_epfd; + +/******************************************************************************* + * Fd Declarations + */ + +struct grpc_fd { + int fd; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Pollset Declarations + */ + +typedef struct pollset_worker_link { + grpc_pollset_worker *next; + grpc_pollset_worker *prev; +} pollset_worker_link; + +typedef enum { + PWL_POLLSET, + PWL_POLLABLE, + POLLSET_WORKER_LINK_COUNT +} pollset_worker_links; + +struct grpc_pollset_worker { + bool kicked; + bool initialized_cv; + pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; + gpr_cv cv; +}; + +struct grpc_pollset { + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set {}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->po.mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->po.mu); + } + + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->po.mu); + new_fd->po.pi = NULL; +#ifdef PO_DEBUG + new_fd->po.obj_type = POLL_OBJ_FD; +#endif + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->po.mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->po.mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->po.mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->po.mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->po.pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->po.pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->po.pi != NULL) { + polling_island *pi_latest = polling_island_lock(fd->po.pi); + polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi_latest->mu); + + unref_pi = fd->po.pi; + fd->po.pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->po.mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + gpr_mu_lock(&fd->po.mu); + grpc_workqueue *workqueue = + GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); + gpr_mu_unlock(&fd->po.mu); + return workqueue; +} + +/******************************************************************************* + * Pollset Definitions + */ + +/* Return true if first in list */ +static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, + grpc_pollset_worker *worker) { + if (*root == NULL) { + *root = worker; + worker->links[link].next = worker->links[link].prev = worker; + return true; + } else { + worker->links[link].next = *root; + worker->links[link].prev = worker->links[link].next->links[link].prev; + worker->links[link].next->links[link].prev = worker; + worker->links[link].prev->links[link].next = worker; + return false; + } +} + +/* Return true if last in list */ +typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; + +static worker_remove_result worker_remove(grpc_pollset_worker **root, + pollset_worker_links link, + grpc_pollset_worker *worker) { + if (worker == *root) { + if (worker == worker->links[link].next) { + *root = NULL; + return EMPTIED; + } else { + *root = worker->links[link].next; + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return NEW_ROOT; + } + } else { + worker->links[link].prev->links[link].next = worker->links[link].next; + worker->links[link].next->links[link].prev = worker->links[link].prev; + return REMOVED; + } +} + +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + return (grpc_pollset_set *)((intptr_t)0xdeafbeef); +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) {} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) {} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) {} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) {} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { + /* If use of signals is disabled, we cannot use epoll engine*/ + if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { + return NULL; + } + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + if (!is_grpc_wakeup_signal_initialized) { + grpc_use_signal(SIGRTMIN + 6); + } + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h new file mode 100644 index 00000000000..5d3651019b3 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -0,0 +1,50 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +// a polling engine that utilizes a singleton epoll set and turnstile polling + +const grpc_event_engine_vtable *grpc_init_epoll1_linux(void); + +#ifdef GRPC_LINUX_EPOLL +void *grpc_fd_get_polling_island(grpc_fd *fd); +void *grpc_pollset_get_polling_island(grpc_pollset *ps); +bool grpc_are_polling_islands_equal(void *p, void *q); +#endif /* defined(GRPC_LINUX_EPOLL) */ + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c similarity index 99% rename from src/core/lib/iomgr/ev_epoll_linux.c rename to src/core/lib/iomgr/ev_epollsig_linux.c index e603a755937..ee2af081f06 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1921,7 +1921,7 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { +const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { return NULL; @@ -1936,7 +1936,7 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { } if (!is_grpc_wakeup_signal_initialized) { - grpc_use_signal(SIGRTMIN + 6); + return NULL; } fd_global_init(); @@ -1958,7 +1958,7 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ void grpc_use_signal(int signum) {} diff --git a/src/core/lib/iomgr/ev_epoll_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h similarity index 89% rename from src/core/lib/iomgr/ev_epoll_linux.h rename to src/core/lib/iomgr/ev_epollsig_linux.h index 8fc3ff59a3e..98a7d9ad920 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.h +++ b/src/core/lib/iomgr/ev_epollsig_linux.h @@ -31,13 +31,13 @@ * */ -#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H -#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epoll_linux(void); +const grpc_event_engine_vtable *grpc_init_epollsig_linux(void); #ifdef GRPC_LINUX_EPOLL void *grpc_fd_get_polling_island(grpc_fd *fd); @@ -45,4 +45,4 @@ void *grpc_pollset_get_polling_island(grpc_pollset *ps); bool grpc_are_polling_islands_equal(void *p, void *q); #endif /* defined(GRPC_LINUX_EPOLL) */ -#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LINUX_H */ +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLSIG_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de83..0e1a93cec42 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,7 +44,8 @@ #include #include -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epoll1_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -65,7 +66,8 @@ typedef struct { } event_engine_factory; static const event_engine_factory g_factories[] = { - {"epoll", grpc_init_epoll_linux}, + {"epollsig", grpc_init_epollsig_linux}, + {"epoll1", grpc_init_epoll1_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 163d90f1a7a..1c3b06dc970 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -97,7 +97,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll1_linux.c', + 'src/core/lib/iomgr/ev_epollsig_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f4..34fc1392292 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -939,8 +939,10 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ -src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll1_linux.c \ +src/core/lib/iomgr/ev_epoll1_linux.h \ +src/core/lib/iomgr/ev_epollsig_linux.c \ +src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a363adc2680..f75edda7f9e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1074,8 +1074,10 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ -src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll1_linux.c \ +src/core/lib/iomgr/ev_epoll1_linux.h \ +src/core/lib/iomgr/ev_epollsig_linux.c \ +src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 386cddb5839..52efa679f56 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7728,7 +7728,8 @@ "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", - "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll1_linux.h", + "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -7869,8 +7870,10 @@ "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", - "src/core/lib/iomgr/ev_epoll_linux.c", - "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll1_linux.c", + "src/core/lib/iomgr/ev_epoll1_linux.h", + "src/core/lib/iomgr/ev_epollsig_linux.c", + "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a588..2dd59e94d1f 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -396,7 +396,8 @@ - + + @@ -610,7 +611,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc2979..a77a4036369 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -184,7 +184,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -908,7 +911,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651c..da167863190 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -390,7 +390,8 @@ - + + @@ -594,7 +595,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab36..8f3c6e0395a 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -169,7 +169,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -875,7 +878,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index d8b61b8c9d0..285dfe4e94c 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -320,7 +320,8 @@ - + + @@ -548,7 +549,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index e1435f4084e..e2f94edbec7 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -64,7 +64,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -866,7 +869,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97c..87aaa3068f4 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -215,7 +215,8 @@ - + + @@ -383,7 +384,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4d..c0400fbbcbd 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -121,7 +121,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -623,7 +626,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0bfda72e815..e0c7057be01 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -310,7 +310,8 @@ - + + @@ -516,7 +517,9 @@ - + + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 63c8d7f2542..0b1bb8ba80b 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -67,7 +67,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr @@ -779,7 +782,10 @@ src\core\lib\iomgr - + + src\core\lib\iomgr + + src\core\lib\iomgr From 4509c476148ae2f3ad56561a6c65bd8b10f110a8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 19:05:13 +0000 Subject: [PATCH 077/177] Fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 467 ++++++++++++++++--------- src/core/lib/iomgr/ev_epollsig_linux.c | 2 +- 2 files changed, 299 insertions(+), 170 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 3c7a16b5ed4..669a3df5c59 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -36,7 +36,7 @@ /* This polling engine is only relevant on linux kernels supporting epoll() */ #ifdef GRPC_LINUX_EPOLL -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epoll1_linux.h" #include #include @@ -75,16 +75,10 @@ static int g_epfd; struct grpc_fd { int fd; - /* The fd is either closed or we relinquished control of it. In either - cases, this indicates that the 'fd' on this structure is no longer - valid */ - bool orphaned; - gpr_atm read_closure; gpr_atm write_closure; struct grpc_fd *freelist_next; - grpc_closure *on_done_closure; /* The pollset that last noticed that the fd is readable. The actual type * stored in this is (grpc_pollset *) */ @@ -119,12 +113,12 @@ struct grpc_pollset_worker { }; struct grpc_pollset { - grpc_pollset_worker root_worker; - bool kicked_without_pollers; + grpc_pollset_worker *root_worker; + bool kicked_without_poller; bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ - grpc_closure *shutdown_done; /* Called after after shutdown is complete */ + grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ }; /******************************************************************************* @@ -171,66 +165,6 @@ static bool append_error(grpc_error **composite, grpc_error *error, static grpc_fd *fd_freelist = NULL; static gpr_mu fd_freelist_mu; -#ifdef GRPC_FD_REF_COUNT_DEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd *fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} - -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - /* Add the fd to the freelist */ - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); - - gpr_mu_unlock(&fd_freelist_mu); - } else { - GPR_ASSERT(old > n); - } -} - -/* Increment refcount by two to avoid changing the orphan bit */ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } -static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } -#endif - static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } static void fd_global_shutdown(void) { @@ -239,7 +173,6 @@ static void fd_global_shutdown(void) { while (fd_freelist != NULL) { grpc_fd *fd = fd_freelist; fd_freelist = fd_freelist->freelist_next; - gpr_mu_destroy(&fd->po.mu); gpr_free(fd); } gpr_mu_destroy(&fd_freelist_mu); @@ -257,29 +190,20 @@ static grpc_fd *fd_create(int fd, const char *name) { if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); - gpr_mu_init(&new_fd->po.mu); } - /* Note: It is not really needed to get the new_fd->po.mu lock here. If this - * is a newly created fd (or an fd we got from the freelist), no one else - * would be holding a lock to it anyway. */ - gpr_mu_lock(&new_fd->po.mu); - new_fd->po.pi = NULL; -#ifdef PO_DEBUG - new_fd->po.obj_type = POLL_OBJ_FD; -#endif + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), + .data.ptr = new_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { + gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); + } - gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; - new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = NULL; - new_fd->on_done_closure = NULL; - - gpr_mu_unlock(&new_fd->po.mu); char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); @@ -291,26 +215,12 @@ static grpc_fd *fd_create(int fd, const char *name) { return new_fd; } -static int fd_wrapped_fd(grpc_fd *fd) { - int ret_fd = -1; - gpr_mu_lock(&fd->po.mu); - if (!fd->orphaned) { - ret_fd = fd->fd; - } - gpr_mu_unlock(&fd->po.mu); - - return ret_fd; -} +static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; } static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, int *release_fd, const char *reason) { - bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - polling_island *unref_pi = NULL; - - gpr_mu_lock(&fd->po.mu); - fd->on_done_closure = on_done; /* If release_fd is not NULL, we should be relinquishing control of the file descriptor fd->fd (but we still own the grpc_fd structure). */ @@ -318,45 +228,18 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, *release_fd = fd->fd; } else { close(fd->fd); - is_fd_closed = true; } - fd->orphaned = true; - - /* Remove the active status but keep referenced. We want this grpc_fd struct - to be alive (and not added to freelist) until the end of this function */ - REF_BY(fd, 1, reason); - - /* Remove the fd from the polling island: - - Get a lock on the latest polling island (i.e the last island in the - linked list pointed by fd->po.pi). This is the island that - would actually contain the fd - - Remove the fd from the latest polling island - - Unlock the latest polling island - - Set fd->po.pi to NULL (but remove the ref on the polling island - before doing this.) */ - if (fd->po.pi != NULL) { - polling_island *pi_latest = polling_island_lock(fd->po.pi); - polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); - gpr_mu_unlock(&pi_latest->mu); - - unref_pi = fd->po.pi; - fd->po.pi = NULL; - } + grpc_closure_sched(exec_ctx, on_done, GRPC_ERROR_REF(error)); - grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + grpc_iomgr_unregister_object(&fd->iomgr_object); + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); - gpr_mu_unlock(&fd->po.mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ - if (unref_pi != NULL) { - /* Unref stale polling island here, outside the fd lock above. - The polling island owns a workqueue which owns an fd, and unreffing - inside the lock can cause an eventual lock loop that makes TSAN very - unhappy. */ - PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); - } - GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); - GRPC_ERROR_UNREF(error); + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + gpr_mu_unlock(&fd_freelist_mu); } static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, @@ -390,11 +273,24 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - gpr_mu_lock(&fd->po.mu); - grpc_workqueue *workqueue = - GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); - gpr_mu_unlock(&fd->po.mu); - return workqueue; + return NULL; /* TODO(ctiller): add a global workqueue */ +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } /******************************************************************************* @@ -442,6 +338,263 @@ static worker_remove_result worker_remove(grpc_pollset_worker **root, GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); +static gpr_mu g_pollset_mu; +static grpc_pollset_worker *g_root_worker; + +static grpc_error *pollset_global_init(void) { + gpr_mu_init(&g_pollset_mu); + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), + .data.ptr = &global_wakeup_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { + return GRPC_OS_ERROR(errno, "epoll_ctl"); + } + return GRPC_ERROR_NONE; +} + +static void pollset_global_shutdown(void) { + gpr_mu_destroy(&g_pollset_mu); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + *mu = &g_pollset_mu; +} + +static grpc_error *pollset_kick_all(grpc_pollset *pollset) { + grpc_error *error = GRPC_ERROR_NONE; + if (pollset->root_worker != NULL) { + grpc_pollset_worker *worker = pollset->root_worker; + do { + if (worker->initialized_cv) { + worker->kicked = true; + gpr_cv_signal(&worker->cv); + } else { + append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), + "pollset_shutdown"); + } + + worker = worker->links[PWL_POLLSET].next; + } while (worker != pollset->root_worker); + } + return error; +} + +static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { + grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); + pollset->shutdown_closure = NULL; + } +} + +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_ASSERT(pollset->shutdown_closure == NULL); + pollset->shutdown_closure = closure; + GRPC_LOG_IF_ERROR("pollset_shutdown", pollset_kick_all(pollset)); + pollset_maybe_finish_shutdown(exec_ctx, pollset); +} + +static void pollset_destroy(grpc_pollset *pollset) {} + +#define MAX_EPOLL_EVENTS 100 + +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, now) <= 0) { + return 0; + } + + static const gpr_timespec round_up = { + .clock_type = GPR_TIMESPAN, .tv_sec = 0, .tv_nsec = GPR_NS_PER_MS - 1}; + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add(timeout, round_up)); + return millis >= 1 ? millis : 1; +} + +static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + gpr_timespec now, gpr_timespec deadline) { + struct epoll_event events[MAX_EPOLL_EVENTS]; + static const char *err_desc = "pollset_poll"; + + int timeout = poll_deadline_to_millis_timeout(deadline, now); + + if (timeout != 0) { + GRPC_SCHEDULING_START_BLOCKING_REGION; + } + int r; + do { + r = epoll_wait(g_epfd, events, MAX_EPOLL_EVENTS, timeout); + } while (r < 0 && errno == EINTR); + if (timeout != 0) { + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); + + grpc_error *error = GRPC_ERROR_NONE; + for (int i = 0; i < r; i++) { + void *data_ptr = events[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else { + grpc_fd *fd = (grpc_fd *)(data_ptr); + bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; + bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; + bool write_ev = (events[i].events & EPOLLOUT) != 0; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + + return error; +} + +static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl, gpr_timespec *now, + gpr_timespec deadline) { + bool do_poll = true; + if (worker_hdl != NULL) *worker_hdl = worker; + worker->initialized_cv = false; + worker->kicked = false; + + worker_insert(&pollset->root_worker, PWL_POLLSET, worker); + if (!worker_insert(&g_root_worker, PWL_POLLABLE, worker)) { + worker->initialized_cv = true; + gpr_cv_init(&worker->cv); + while (do_poll && g_root_worker != worker) { + if (gpr_cv_wait(&worker->cv, &g_pollset_mu, deadline)) { + do_poll = false; + } else if (worker->kicked) { + do_poll = false; + } + } + *now = gpr_now(now->clock_type); + } + + return do_poll && pollset->shutdown_closure == NULL; +} + +static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker *worker, + grpc_pollset_worker **worker_hdl) { + if (NEW_ROOT == worker_remove(&g_root_worker, PWL_POLLABLE, worker)) { + gpr_cv_signal(&g_root_worker->cv); + } + if (worker->initialized_cv) { + gpr_cv_destroy(&worker->cv); + } + if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + grpc_pollset_worker worker; + grpc_error *error = GRPC_ERROR_NONE; + static const char *err_desc = "pollset_work"; + if (pollset->kicked_without_poller) { + pollset->kicked_without_poller = false; + return GRPC_ERROR_NONE; + } + if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + GPR_ASSERT(!pollset->shutdown_closure); + gpr_mu_unlock(&g_pollset_mu); + append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), + err_desc); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&g_pollset_mu); + gpr_tls_set(&g_current_thread_pollset, 0); + gpr_tls_set(&g_current_thread_worker, 0); + pollset_maybe_finish_shutdown(exec_ctx, pollset); + } + end_worker(exec_ctx, pollset, &worker, worker_hdl); + return error; +} + +static grpc_error *pollset_kick(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker) { + if (specific_worker == NULL) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { + if (pollset->root_worker == NULL) { + pollset->kicked_without_poller = true; + return GRPC_ERROR_NONE; + } else { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } + } else { + return GRPC_ERROR_NONE; + } + } else if (specific_worker->kicked) { + return GRPC_ERROR_NONE; + } else if (gpr_tls_get(&g_current_thread_worker) == + (intptr_t)specific_worker) { + specific_worker->kicked = true; + return GRPC_ERROR_NONE; + } else if (specific_worker == g_root_worker) { + specific_worker->kicked = true; + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } else { + specific_worker->kicked = true; + gpr_cv_signal(&specific_worker->cv); + return GRPC_ERROR_NONE; + } +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) {} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +/******************************************************************************* + * Workqueue Definitions + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) {} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) {} +#endif + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + return grpc_schedule_on_exec_ctx; +} /******************************************************************************* * Pollset-set Definitions @@ -481,7 +634,6 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { fd_global_shutdown(); pollset_global_shutdown(); - polling_island_global_shutdown(); } static const grpc_event_engine_vtable vtable = { @@ -524,45 +676,22 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ -static bool is_epoll_available() { - int fd = epoll_create1(EPOLL_CLOEXEC); - if (fd < 0) { - gpr_log( - GPR_ERROR, - "epoll_create1 failed with error: %d. Not using epoll polling engine", - fd); - return false; - } - close(fd); - return true; -} - const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { - /* If use of signals is disabled, we cannot use epoll engine*/ - if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { - return NULL; - } - if (!grpc_has_wakeup_fd()) { return NULL; } - if (!is_epoll_available()) { + g_epfd = epoll_create1(EPOLL_CLOEXEC); + if (g_epfd < 0) { + gpr_log(GPR_ERROR, "epoll unavailable"); return NULL; } - if (!is_grpc_wakeup_signal_initialized) { - grpc_use_signal(SIGRTMIN + 6); - } - fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { - return NULL; - } - - if (!GRPC_LOG_IF_ERROR("polling_island_global_init", - polling_island_global_init())) { + close(g_epfd); + fd_global_shutdown(); return NULL; } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index ee2af081f06..4ce380c7d0e 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -36,7 +36,7 @@ /* This polling engine is only relevant on linux kernels supporting epoll() */ #ifdef GRPC_LINUX_EPOLL -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include #include From b72a74ae039d666cf6cfc60477289c03dc5b5933 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 12:07:05 -0700 Subject: [PATCH 078/177] Fix wakeup bug --- src/core/lib/iomgr/ev_epollex_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b7c3e2e959f..952f0172ccf 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -696,7 +696,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, p->root_worker); } if (specific_worker == NULL) { - if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { if (pollset->root_worker == NULL) { if (grpc_polling_trace) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); From d4838a98c5fb8b1227420f7577615ef7f66ac43e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 12:08:18 -0700 Subject: [PATCH 079/177] Tweaks --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- src/core/lib/iomgr/ev_epoll1_linux.h | 8 +------- tools/run_tests/run_tests.py | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 3c7a16b5ed4..67edae68a6e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -1,6 +1,6 @@ /* * - * Copyright 2016, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h index 5d3651019b3..99f48270083 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.h +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -41,10 +41,4 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(void); -#ifdef GRPC_LINUX_EPOLL -void *grpc_fd_get_polling_island(grpc_fd *fd); -void *grpc_pollset_get_polling_island(grpc_pollset *ps); -bool grpc_are_polling_islands_equal(void *p, void *q); -#endif /* defined(GRPC_LINUX_EPOLL) */ - #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4da2ba4c3b0..c1da93f4229 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -72,7 +72,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { _POLLING_STRATEGIES = { - 'linux': ['epoll', 'poll', 'poll-cv'] + 'linux': ['epoll1', 'epollsig', 'poll', 'poll-cv'] } From 6a85ee0b4d32ec0999162dd419e05f3f2f93c083 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 19:17:11 +0000 Subject: [PATCH 080/177] Fix test compilation --- CMakeLists.txt | 10 +++---- Makefile | 26 +++++++++---------- build.yaml | 4 +-- ..._linux_test.c => ev_epollsig_linux_test.c} | 6 ++--- .../generated/sources_and_headers.json | 4 +-- tools/run_tests/generated/tests.json | 2 +- 6 files changed, 26 insertions(+), 26 deletions(-) rename test/core/iomgr/{ev_epoll_linux_test.c => ev_epollsig_linux_test.c} (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2debe0422e7..8a72d3abccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -391,7 +391,7 @@ endif() add_dependencies(buildtests_c endpoint_pair_test) add_dependencies(buildtests_c error_test) if(_gRPC_PLATFORM_LINUX) -add_dependencies(buildtests_c ev_epoll_linux_test) +add_dependencies(buildtests_c ev_epollsig_linux_test) endif() add_dependencies(buildtests_c fake_resolver_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -5423,12 +5423,12 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX) -add_executable(ev_epoll_linux_test - test/core/iomgr/ev_epoll_linux_test.c +add_executable(ev_epollsig_linux_test + test/core/iomgr/ev_epollsig_linux_test.c ) -target_include_directories(ev_epoll_linux_test +target_include_directories(ev_epollsig_linux_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${BORINGSSL_ROOT_DIR}/include @@ -5443,7 +5443,7 @@ target_include_directories(ev_epoll_linux_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include ) -target_link_libraries(ev_epoll_linux_test +target_link_libraries(ev_epollsig_linux_test ${_gRPC_ALLTARGETS_LIBRARIES} grpc_test_util grpc diff --git a/Makefile b/Makefile index b5d8b5b2549..ab99e8e2796 100644 --- a/Makefile +++ b/Makefile @@ -991,7 +991,7 @@ dns_resolver_test: $(BINDIR)/$(CONFIG)/dns_resolver_test dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test error_test: $(BINDIR)/$(CONFIG)/error_test -ev_epoll_linux_test: $(BINDIR)/$(CONFIG)/ev_epoll_linux_test +ev_epollsig_linux_test: $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test fake_resolver_test: $(BINDIR)/$(CONFIG)/fake_resolver_test fd_conservation_posix_test: $(BINDIR)/$(CONFIG)/fd_conservation_posix_test fd_posix_test: $(BINDIR)/$(CONFIG)/fd_posix_test @@ -1379,7 +1379,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/dualstack_socket_test \ $(BINDIR)/$(CONFIG)/endpoint_pair_test \ $(BINDIR)/$(CONFIG)/error_test \ - $(BINDIR)/$(CONFIG)/ev_epoll_linux_test \ + $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test \ $(BINDIR)/$(CONFIG)/fake_resolver_test \ $(BINDIR)/$(CONFIG)/fd_conservation_posix_test \ $(BINDIR)/$(CONFIG)/fd_posix_test \ @@ -1793,8 +1793,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/endpoint_pair_test || ( echo test endpoint_pair_test failed ; exit 1 ) $(E) "[RUN] Testing error_test" $(Q) $(BINDIR)/$(CONFIG)/error_test || ( echo test error_test failed ; exit 1 ) - $(E) "[RUN] Testing ev_epoll_linux_test" - $(Q) $(BINDIR)/$(CONFIG)/ev_epoll_linux_test || ( echo test ev_epoll_linux_test failed ; exit 1 ) + $(E) "[RUN] Testing ev_epollsig_linux_test" + $(Q) $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test || ( echo test ev_epollsig_linux_test failed ; exit 1 ) $(E) "[RUN] Testing fake_resolver_test" $(Q) $(BINDIR)/$(CONFIG)/fake_resolver_test || ( echo test fake_resolver_test failed ; exit 1 ) $(E) "[RUN] Testing fd_conservation_posix_test" @@ -9414,34 +9414,34 @@ endif endif -EV_EPOLL_LINUX_TEST_SRC = \ - test/core/iomgr/ev_epoll_linux_test.c \ +EV_EPOLLSIG_LINUX_TEST_SRC = \ + test/core/iomgr/ev_epollsig_linux_test.c \ -EV_EPOLL_LINUX_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(EV_EPOLL_LINUX_TEST_SRC)))) +EV_EPOLLSIG_LINUX_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(EV_EPOLLSIG_LINUX_TEST_SRC)))) ifeq ($(NO_SECURE),true) # You can't build secure targets if you don't have OpenSSL. -$(BINDIR)/$(CONFIG)/ev_epoll_linux_test: openssl_dep_error +$(BINDIR)/$(CONFIG)/ev_epollsig_linux_test: openssl_dep_error else -$(BINDIR)/$(CONFIG)/ev_epoll_linux_test: $(EV_EPOLL_LINUX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(BINDIR)/$(CONFIG)/ev_epollsig_linux_test: $(EV_EPOLLSIG_LINUX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(EV_EPOLL_LINUX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/ev_epoll_linux_test + $(Q) $(LD) $(LDFLAGS) $(EV_EPOLLSIG_LINUX_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/ev_epollsig_linux_test endif -$(OBJDIR)/$(CONFIG)/test/core/iomgr/ev_epoll_linux_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a +$(OBJDIR)/$(CONFIG)/test/core/iomgr/ev_epollsig_linux_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a -deps_ev_epoll_linux_test: $(EV_EPOLL_LINUX_TEST_OBJS:.o=.dep) +deps_ev_epollsig_linux_test: $(EV_EPOLLSIG_LINUX_TEST_OBJS:.o=.dep) ifneq ($(NO_SECURE),true) ifneq ($(NO_DEPS),true) --include $(EV_EPOLL_LINUX_TEST_OBJS:.o=.dep) +-include $(EV_EPOLLSIG_LINUX_TEST_OBJS:.o=.dep) endif endif diff --git a/build.yaml b/build.yaml index 89b356c8f2c..aed56e12ac8 100644 --- a/build.yaml +++ b/build.yaml @@ -1814,12 +1814,12 @@ targets: - grpc - gpr_test_util - gpr -- name: ev_epoll_linux_test +- name: ev_epollsig_linux_test cpu_cost: 3 build: test language: c src: - - test/core/iomgr/ev_epoll_linux_test.c + - test/core/iomgr/ev_epollsig_linux_test.c deps: - grpc_test_util - grpc diff --git a/test/core/iomgr/ev_epoll_linux_test.c b/test/core/iomgr/ev_epollsig_linux_test.c similarity index 99% rename from test/core/iomgr/ev_epoll_linux_test.c rename to test/core/iomgr/ev_epollsig_linux_test.c index 0856023b14f..69695b8b44e 100644 --- a/test/core/iomgr/ev_epoll_linux_test.c +++ b/test/core/iomgr/ev_epollsig_linux_test.c @@ -34,7 +34,7 @@ /* This test only relevant on linux systems where epoll() is available */ #ifdef GRPC_LINUX_EPOLL -#include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_posix.h" #include @@ -405,13 +405,13 @@ int main(int argc, char **argv) { grpc_iomgr_init(); poll_strategy = grpc_get_poll_strategy_name(); - if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { + if (poll_strategy != NULL && strcmp(poll_strategy, "epollsig") == 0) { test_add_fd_to_pollset(); test_pollset_queue_merge_items(); test_threading(); } else { gpr_log(GPR_INFO, - "Skipping the test. The test is only relevant for 'epoll' " + "Skipping the test. The test is only relevant for 'epollsig' " "strategy. and the current strategy is: '%s'", poll_strategy); } diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 52efa679f56..27f772fc574 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -463,9 +463,9 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "ev_epoll_linux_test", + "name": "ev_epollsig_linux_test", "src": [ - "test/core/iomgr/ev_epoll_linux_test.c" + "test/core/iomgr/ev_epollsig_linux_test.c" ], "third_party": false, "type": "target" diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 17f7c4b454e..4b8f64db5b1 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -572,7 +572,7 @@ "flaky": false, "gtest": false, "language": "c", - "name": "ev_epoll_linux_test", + "name": "ev_epollsig_linux_test", "platforms": [ "linux" ] From 6f0af49bb2f784a66d5fcc9cf9ba46754451f7e9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 19:26:16 +0000 Subject: [PATCH 081/177] Allow specifying a default signal iff epollsig has been explicitly requested --- src/core/lib/iomgr/ev_epoll1_linux.c | 4 ++-- src/core/lib/iomgr/ev_epoll1_linux.h | 2 +- src/core/lib/iomgr/ev_epollsig_linux.c | 7 ++++--- src/core/lib/iomgr/ev_epollsig_linux.h | 2 +- src/core/lib/iomgr/ev_poll_posix.c | 4 ++-- src/core/lib/iomgr/ev_poll_posix.h | 4 ++-- src/core/lib/iomgr/ev_posix.c | 4 ++-- 7 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index ff4c51f5d89..9909d8b126c 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -676,7 +676,7 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ -const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -703,6 +703,6 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll1_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll1_linux.h b/src/core/lib/iomgr/ev_epoll1_linux.h index 99f48270083..bd52478a7c1 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.h +++ b/src/core/lib/iomgr/ev_epoll1_linux.h @@ -39,6 +39,6 @@ // a polling engine that utilizes a singleton epoll set and turnstile polling -const grpc_event_engine_vtable *grpc_init_epoll1_linux(void); +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL1_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 4ce380c7d0e..25986598b78 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1921,7 +1921,7 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { return NULL; @@ -1936,7 +1936,8 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { } if (!is_grpc_wakeup_signal_initialized) { - return NULL; + if (explicit_request) grpc_use_signal(SIGRTMIN + 6); + else return NULL; } fd_global_init(); @@ -1958,7 +1959,7 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epollsig_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ void grpc_use_signal(int signum) {} diff --git a/src/core/lib/iomgr/ev_epollsig_linux.h b/src/core/lib/iomgr/ev_epollsig_linux.h index 98a7d9ad920..9e4034f2a7f 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.h +++ b/src/core/lib/iomgr/ev_epollsig_linux.h @@ -37,7 +37,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epollsig_linux(void); +const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request); #ifdef GRPC_LINUX_EPOLL void *grpc_fd_get_polling_island(grpc_fd *fd); diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 9834cdd1979..69f322f7df9 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -1569,7 +1569,7 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -const grpc_event_engine_vtable *grpc_init_poll_posix(void) { +const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1579,7 +1579,7 @@ const grpc_event_engine_vtable *grpc_init_poll_posix(void) { return &vtable; } -const grpc_event_engine_vtable *grpc_init_poll_cv_posix(void) { +const grpc_event_engine_vtable *grpc_init_poll_cv_posix(bool explicit_request) { global_cv_fd_table_init(); grpc_enable_cv_wakeup_fds(1); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { diff --git a/src/core/lib/iomgr/ev_poll_posix.h b/src/core/lib/iomgr/ev_poll_posix.h index 202ffca14ca..2890e93eade 100644 --- a/src/core/lib/iomgr/ev_poll_posix.h +++ b/src/core/lib/iomgr/ev_poll_posix.h @@ -36,7 +36,7 @@ #include "src/core/lib/iomgr/ev_posix.h" -const grpc_event_engine_vtable *grpc_init_poll_posix(void); -const grpc_event_engine_vtable *grpc_init_poll_cv_posix(void); +const grpc_event_engine_vtable *grpc_init_poll_posix(bool explicit_request); +const grpc_event_engine_vtable *grpc_init_poll_cv_posix(bool explicit_request); #endif /* GRPC_CORE_LIB_IOMGR_EV_POLL_POSIX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 0e1a93cec42..442faeac3eb 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -58,7 +58,7 @@ grpc_wakeup_fd grpc_global_wakeup_fd; static const grpc_event_engine_vtable *g_event_engine; static const char *g_poll_strategy_name = NULL; -typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(void); +typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(bool explicit_request); typedef struct { const char *name; @@ -104,7 +104,7 @@ static bool is(const char *want, const char *have) { static void try_engine(const char *engine) { for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { if (is(engine, g_factories[i].name)) { - if ((g_event_engine = g_factories[i].factory())) { + if ((g_event_engine = g_factories[i].factory(0 == strcmp(engine, g_factories[i].name)))) { g_poll_strategy_name = g_factories[i].name; gpr_log(GPR_DEBUG, "Using polling engine: %s", g_factories[i].name); return; From 9ddb3151d6ae32027f103322d8ff984ea3eecefa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 21:32:56 +0000 Subject: [PATCH 082/177] Bug fixes --- src/core/lib/iomgr/ev_epoll1_linux.c | 41 ++++++++++++++++------------ 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 9909d8b126c..d8d3cb63766 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -192,12 +192,6 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd = gpr_malloc(sizeof(grpc_fd)); } - struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), - .data.ptr = new_fd}; - if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { - gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); - } - new_fd->fd = fd; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); @@ -212,16 +206,37 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); #endif gpr_free(fd_name); + + struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET), + .data.ptr = new_fd}; + if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, fd, &ev) != 0) { + gpr_log(GPR_ERROR, "epoll_ctl failed: %s", strerror(errno)); + } + return new_fd; } static int fd_wrapped_fd(grpc_fd *fd) { return fd->fd; } +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done, int *release_fd, const char *reason) { grpc_error *error = GRPC_ERROR_NONE; + if (!grpc_lfev_is_shutdown(&fd->read_closure)) { + fd_shutdown(exec_ctx, fd, GRPC_ERROR_CREATE_FROM_COPIED_STRING(reason)); + } + /* If release_fd is not NULL, we should be relinquishing control of the file descriptor fd->fd (but we still own the grpc_fd structure). */ if (release_fd != NULL) { @@ -252,16 +267,6 @@ static bool fd_is_shutdown(grpc_fd *fd) { return grpc_lfev_is_shutdown(&fd->read_closure); } -/* Might be called multiple times */ -static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { - if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, - GRPC_ERROR_REF(why))) { - shutdown(fd->fd, SHUT_RDWR); - grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); - } - GRPC_ERROR_UNREF(why); -} - static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *closure) { grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); @@ -703,6 +708,8 @@ const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ From f8382b80caf5f27a69f12e8620cc27255f8afdb8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 15:09:48 -0700 Subject: [PATCH 083/177] clang-format --- src/core/lib/iomgr/ev_epollsig_linux.c | 15 +++++++++++---- src/core/lib/iomgr/ev_posix.c | 6 ++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 25986598b78..65259912a9a 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1921,7 +1921,8 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { +const grpc_event_engine_vtable *grpc_init_epollsig_linux( + bool explicit_request) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { return NULL; @@ -1936,8 +1937,11 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) } if (!is_grpc_wakeup_signal_initialized) { - if (explicit_request) grpc_use_signal(SIGRTMIN + 6); - else return NULL; + if (explicit_request) { + grpc_use_signal(SIGRTMIN + 6); + } else { + return NULL; + } } fd_global_init(); @@ -1959,7 +1963,10 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epollsig_linux(bool explicit_request) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollsig_linux( + bool explicit_request) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ void grpc_use_signal(int signum) {} diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 442faeac3eb..a436d649dd2 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -58,7 +58,8 @@ grpc_wakeup_fd grpc_global_wakeup_fd; static const grpc_event_engine_vtable *g_event_engine; static const char *g_poll_strategy_name = NULL; -typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)(bool explicit_request); +typedef const grpc_event_engine_vtable *(*event_engine_factory_fn)( + bool explicit_request); typedef struct { const char *name; @@ -104,7 +105,8 @@ static bool is(const char *want, const char *have) { static void try_engine(const char *engine) { for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { if (is(engine, g_factories[i].name)) { - if ((g_event_engine = g_factories[i].factory(0 == strcmp(engine, g_factories[i].name)))) { + if ((g_event_engine = g_factories[i].factory( + 0 == strcmp(engine, g_factories[i].name)))) { g_poll_strategy_name = g_factories[i].name; gpr_log(GPR_DEBUG, "Using polling engine: %s", g_factories[i].name); return; From 375eb258d8d66dc3ff1e8a9e82760b8ce6e51954 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 27 Apr 2017 23:29:12 +0000 Subject: [PATCH 084/177] Fix kicks --- src/core/lib/iomgr/ev_epoll1_linux.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index d8d3cb63766..195c3bd1441 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -67,6 +67,7 @@ * needed) */ static grpc_wakeup_fd global_wakeup_fd; static int g_epfd; +static bool g_timer_kick = false; /******************************************************************************* * Fd Declarations @@ -350,6 +351,9 @@ static grpc_error *pollset_global_init(void) { gpr_mu_init(&g_pollset_mu); gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); + global_wakeup_fd.read_fd = -1; + grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); + if (err != GRPC_ERROR_NONE) return err; struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &global_wakeup_fd}; if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { @@ -362,6 +366,7 @@ static void pollset_global_shutdown(void) { gpr_mu_destroy(&g_pollset_mu); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); + if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -449,7 +454,10 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); + if (g_timer_kick) { + g_timer_kick = false; + grpc_timer_consume_kick(); + } append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else { @@ -543,11 +551,19 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { - if (pollset->root_worker == NULL) { + grpc_pollset_worker *root_worker = pollset->root_worker; + if (root_worker == NULL) { pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; - } else { + } + grpc_pollset_worker *next_worker = root_worker->links[PWL_POLLSET].next; + if (root_worker == next_worker && root_worker == g_root_worker) { + root_worker->kicked = true; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); + } else { + next_worker->kicked = true; + gpr_cv_signal(&next_worker->cv); + return GRPC_ERROR_NONE; } } else { return GRPC_ERROR_NONE; @@ -572,6 +588,9 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) {} static grpc_error *kick_poller(void) { + gpr_mu_lock(&g_pollset_mu); + g_timer_kick = true; + gpr_mu_unlock(&g_pollset_mu); return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } From 2f128f9a0b304ad740ddf26c7b25494d5a9cb6c6 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Thu, 27 Apr 2017 16:40:43 -0700 Subject: [PATCH 085/177] add blocking region annotations around sigtimedwait --- src/core/lib/iomgr/ev_epoll_linux.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 1234828063b..64ea7b192ac 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1489,7 +1489,9 @@ static bool acquire_polling_lease(grpc_pollset_worker *worker, ret = sigwaitinfo(&g_wakeup_sig_set, NULL); } else { struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + GRPC_SCHEDULING_START_BLOCKING_REGION; ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; } if (ret == -1) { From 6de0593e825dc616924847f8a1583b3badaaa3fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 09:17:38 -0700 Subject: [PATCH 086/177] Start sketching hierarchical turnstile --- src/core/lib/iomgr/ev_epoll1_linux.c | 53 ++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 195c3bd1441..fdd6384c867 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -113,18 +114,32 @@ struct grpc_pollset_worker { gpr_cv cv; }; +typedef struct pollset_neighbourhood { + gpr_mu mu; + grpc_pollset *active_root; + grpc_pollset *inactive_root; + bool seen_inactive; + char pad[GPR_CACHELINE_SIZE]; +} pollset_neighbourhood; + struct grpc_pollset { + gpr_mu mu; + pollset_neighbourhood *neighbourhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; - + bool seen_inactive; bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ + + grpc_pollset *next; + grpc_pollset *prev; }; /******************************************************************************* * Pollset-set Declarations */ + struct grpc_pollset_set {}; /******************************************************************************* @@ -303,6 +318,11 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { * Pollset Definitions */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static gpr_atm g_active_poller; +static pollset_neighbourhood *g_neighbourhoods; + /* Return true if first in list */ static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, grpc_pollset_worker *worker) { @@ -342,15 +362,10 @@ static worker_remove_result worker_remove(grpc_pollset_worker **root, } } -GPR_TLS_DECL(g_current_thread_pollset); -GPR_TLS_DECL(g_current_thread_worker); -static gpr_mu g_pollset_mu; -static grpc_pollset_worker *g_root_worker; - static grpc_error *pollset_global_init(void) { - gpr_mu_init(&g_pollset_mu); gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); + gpr_atm_no_barrier_store(&g_active_poller, 0); global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); if (err != GRPC_ERROR_NONE) return err; @@ -363,14 +378,32 @@ static grpc_error *pollset_global_init(void) { } static void pollset_global_shutdown(void) { - gpr_mu_destroy(&g_pollset_mu); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - *mu = &g_pollset_mu; + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset->seen_inactive = true; + pollset->next = pollset->prev = pollset; +} + +static void pollset_destroy(grpc_pollset *pollset) { + gpr_mu_destroy(&pollset->mu); + gpr_mu_lock(&pollset->neighbourhood->mu); + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } else if (pollset == pollset->neighbourhood->inactive_root) { + pollset->neighbourhood->inactive_root = + pollset->next == pollset ? NULL : pollset->next; + } + gpr_mu_unlock(&pollset->neighbourhood->mu); } static grpc_error *pollset_kick_all(grpc_pollset *pollset) { @@ -408,8 +441,6 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -static void pollset_destroy(grpc_pollset *pollset) {} - #define MAX_EPOLL_EVENTS 100 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, From 32f90eec45cd7e19e7742e8868110d945e0eab73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 12:46:41 -0700 Subject: [PATCH 087/177] bug fixes, implementation --- src/core/lib/iomgr/ev_epoll1_linux.c | 245 +++++++++++++++++++-------- 1 file changed, 179 insertions(+), 66 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index fdd6384c867..3355c8344ec 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -68,7 +68,7 @@ * needed) */ static grpc_wakeup_fd global_wakeup_fd; static int g_epfd; -static bool g_timer_kick = false; +static gpr_atm g_timer_kick; /******************************************************************************* * Fd Declarations @@ -96,21 +96,13 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef struct pollset_worker_link { - grpc_pollset_worker *next; - grpc_pollset_worker *prev; -} pollset_worker_link; - -typedef enum { - PWL_POLLSET, - PWL_POLLABLE, - POLLSET_WORKER_LINK_COUNT -} pollset_worker_links; +typedef enum { UNKICKED, KICKED, KICKED_FOR_POLL } kick_state; struct grpc_pollset_worker { - bool kicked; + kick_state kick_state; bool initialized_cv; - pollset_worker_link links[POLLSET_WORKER_LINK_COUNT]; + grpc_pollset_worker *next; + grpc_pollset_worker *prev; gpr_cv cv; }; @@ -322,19 +314,19 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; +static size_t g_num_neighbourhoods; /* Return true if first in list */ -static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, - grpc_pollset_worker *worker) { - if (*root == NULL) { - *root = worker; - worker->links[link].next = worker->links[link].prev = worker; +static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { + if (pollset->root_worker == NULL) { + pollset->root_worker = worker; + worker->next = worker->prev = worker; return true; } else { - worker->links[link].next = *root; - worker->links[link].prev = worker->links[link].next->links[link].prev; - worker->links[link].next->links[link].prev = worker; - worker->links[link].prev->links[link].next = worker; + worker->next = pollset->root_worker; + worker->prev = worker->next->prev; + worker->next->prev = worker; + worker->prev->next = worker; return false; } } @@ -342,22 +334,21 @@ static bool worker_insert(grpc_pollset_worker **root, pollset_worker_links link, /* Return true if last in list */ typedef enum { EMPTIED, NEW_ROOT, REMOVED } worker_remove_result; -static worker_remove_result worker_remove(grpc_pollset_worker **root, - pollset_worker_links link, +static worker_remove_result worker_remove(grpc_pollset *pollset, grpc_pollset_worker *worker) { - if (worker == *root) { - if (worker == worker->links[link].next) { - *root = NULL; + if (worker == pollset->root_worker) { + if (worker == worker->next) { + pollset->root_worker = NULL; return EMPTIED; } else { - *root = worker->links[link].next; - worker->links[link].prev->links[link].next = worker->links[link].next; - worker->links[link].next->links[link].prev = worker->links[link].prev; + pollset->root_worker = worker->next; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; return NEW_ROOT; } } else { - worker->links[link].prev->links[link].next = worker->links[link].next; - worker->links[link].next->links[link].prev = worker->links[link].prev; + worker->prev->next = worker->next; + worker->next->prev = worker->prev; return REMOVED; } } @@ -374,6 +365,13 @@ static grpc_error *pollset_global_init(void) { if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } + g_num_neighbourhoods = GPR_MAX(1, gpr_cpu_num_cores()); + g_neighbourhoods = + gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); + for (size_t i = 0; i < g_num_neighbourhoods; i++) { + gpr_mu_init(&g_neighbourhoods[i].mu); + g_neighbourhoods[i].seen_inactive = true; + } return GRPC_ERROR_NONE; } @@ -381,6 +379,10 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); + for (size_t i = 0; i < g_num_neighbourhoods; i++) { + gpr_mu_destroy(&g_neighbourhoods[i].mu); + } + gpr_free(g_neighbourhoods); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -392,7 +394,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { } static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->mu); gpr_mu_lock(&pollset->neighbourhood->mu); pollset->prev->next = pollset->next; pollset->next->prev = pollset->prev; @@ -404,6 +405,7 @@ static void pollset_destroy(grpc_pollset *pollset) { pollset->next == pollset ? NULL : pollset->next; } gpr_mu_unlock(&pollset->neighbourhood->mu); + gpr_mu_destroy(&pollset->mu); } static grpc_error *pollset_kick_all(grpc_pollset *pollset) { @@ -412,14 +414,15 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { grpc_pollset_worker *worker = pollset->root_worker; do { if (worker->initialized_cv) { - worker->kicked = true; + worker->kick_state = KICKED; gpr_cv_signal(&worker->cv); } else { + worker->kick_state = KICKED; append_error(&error, grpc_wakeup_fd_wakeup(&global_wakeup_fd), "pollset_shutdown"); } - worker = worker->links[PWL_POLLSET].next; + worker = worker->next; } while (worker != pollset->root_worker); } return error; @@ -485,8 +488,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - if (g_timer_kick) { - g_timer_kick = false; + if (gpr_atm_no_barrier_cas(&g_timer_kick, 1, 0)) { grpc_timer_consume_kick(); } append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), @@ -508,41 +510,151 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } +#if 0 +static void verify_all_entries_in_neighbourhood_list( + grpc_pollset *root, bool should_be_seen_inactive) { + if (root == NULL) return; + grpc_pollset *p = root; + do { + GPR_ASSERT(p->seen_inactive == should_be_seen_inactive); + p = p->next; + } while (p != root); +} + +static void verify_neighbourhood_lists(pollset_neighbourhood *neighbourhood) { + // assumes neighbourhood->mu locked + verify_all_entries_in_neighbourhood_list(neighbourhood->active_root, false); + verify_all_entries_in_neighbourhood_list(neighbourhood->inactive_root, true); +} +#endif + +static void move_pollset_to_neighbourhood_list(grpc_pollset *pollset, + grpc_pollset **from_root, + grpc_pollset **to_root) { + // remove from old list + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (*from_root == pollset) { + *from_root = pollset->next == pollset ? NULL : pollset->next; + } + // add to new list + if (*to_root == NULL) { + *to_root = pollset->next = pollset->prev = pollset; + } else { + pollset->next = *to_root; + pollset->prev = pollset->next->prev; + pollset->next->prev = pollset->prev->next = pollset; + } +} + static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { - bool do_poll = true; if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; - worker->kicked = false; - - worker_insert(&pollset->root_worker, PWL_POLLSET, worker); - if (!worker_insert(&g_root_worker, PWL_POLLABLE, worker)) { + worker->kick_state = UNKICKED; + + if (pollset->seen_inactive) { + // pollset has been observed to be inactive, we need to move back to the + // active list + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + gpr_mu_lock(&neighbourhood->mu); + gpr_mu_lock(&pollset->mu); + if (pollset->seen_inactive) { + pollset->seen_inactive = false; + move_pollset_to_neighbourhood_list(pollset, &neighbourhood->inactive_root, + &neighbourhood->active_root); + if (neighbourhood->seen_inactive) { + neighbourhood->seen_inactive = false; + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { + worker->kick_state = KICKED_FOR_POLL; + } + } + } + gpr_mu_unlock(&neighbourhood->mu); + } + worker_insert(pollset, worker); + if (worker->kick_state == UNKICKED) { worker->initialized_cv = true; gpr_cv_init(&worker->cv); - while (do_poll && g_root_worker != worker) { - if (gpr_cv_wait(&worker->cv, &g_pollset_mu, deadline)) { - do_poll = false; - } else if (worker->kicked) { - do_poll = false; + do { + if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && + worker->kick_state == UNKICKED) { + worker->kick_state = KICKED; } - } + } while (worker->kick_state == UNKICKED); *now = gpr_now(now->clock_type); } - return do_poll && pollset->shutdown_closure == NULL; + return worker->kick_state == KICKED_FOR_POLL && + pollset->shutdown_closure == NULL; } static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (NEW_ROOT == worker_remove(&g_root_worker, PWL_POLLABLE, worker)) { - gpr_cv_signal(&g_root_worker->cv); + if (worker->kick_state == KICKED_FOR_POLL) { + GPR_ASSERT(!pollset->seen_inactive); + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker); + if (worker->next != worker) { + assert(worker->next->initialized_cv); + gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); + worker->next->kick_state = KICKED_FOR_POLL; + gpr_cv_signal(&worker->next->cv); + } else { + gpr_atm_no_barrier_store(&g_active_poller, 0); + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + bool found_worker = false; + do { + gpr_mu_lock(&neighbourhood->mu); + do { + grpc_pollset *inspect = neighbourhood->active_root; + if (inspect == NULL) { + break; + } + gpr_mu_lock(&inspect->mu); + GPR_ASSERT(!inspect->seen_inactive); + grpc_pollset_worker *inspect_worker = inspect->root_worker; + if (inspect_worker == worker) inspect_worker = worker->next; + if (inspect_worker == worker) inspect_worker = NULL; + if (inspect_worker != NULL) { + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + GPR_ASSERT(inspect_worker->initialized_cv); + inspect_worker->kick_state = KICKED_FOR_POLL; + gpr_cv_signal(&inspect_worker->cv); + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + } else { + inspect->seen_inactive = true; + move_pollset_to_neighbourhood_list(inspect, + &neighbourhood->active_root, + &neighbourhood->inactive_root); + } + gpr_mu_unlock(&inspect->mu); + } while (!found_worker); + if (!found_worker) { + neighbourhood->seen_inactive = true; + } + gpr_mu_unlock(&neighbourhood->mu); + ssize_t cur_neighbourhood_idx = neighbourhood - g_neighbourhoods; + GPR_ASSERT(cur_neighbourhood_idx >= 0); + GPR_ASSERT(g_num_neighbourhoods < INTPTR_MAX); + GPR_ASSERT(cur_neighbourhood_idx < (ssize_t)g_neighbourhoods); + size_t new_neighbourhood_idx = + ((size_t)cur_neighbourhood_idx + 1) % g_num_neighbourhoods; + neighbourhood = &g_neighbourhoods[new_neighbourhood_idx]; + } while (!found_worker && neighbourhood != pollset->neighbourhood); + gpr_mu_lock(&pollset->mu); + } } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); } - if (EMPTIED == worker_remove(&pollset->root_worker, PWL_POLLSET, worker)) { + if (EMPTIED == worker_remove(pollset, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); } } @@ -565,11 +677,11 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); - gpr_mu_unlock(&g_pollset_mu); + gpr_mu_unlock(&pollset->mu); append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), err_desc); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&g_pollset_mu); + gpr_mu_lock(&pollset->mu); gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); pollset_maybe_finish_shutdown(exec_ctx, pollset); @@ -587,29 +699,32 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; } - grpc_pollset_worker *next_worker = root_worker->links[PWL_POLLSET].next; - if (root_worker == next_worker && root_worker == g_root_worker) { - root_worker->kicked = true; + grpc_pollset_worker *next_worker = root_worker->next; + if (root_worker == next_worker && + root_worker == (grpc_pollset_worker *)gpr_atm_no_barrier_load( + &g_active_poller)) { + root_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } else { - next_worker->kicked = true; + next_worker->kick_state = KICKED; gpr_cv_signal(&next_worker->cv); return GRPC_ERROR_NONE; } } else { return GRPC_ERROR_NONE; } - } else if (specific_worker->kicked) { + } else if (specific_worker->kick_state != KICKED) { return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { - specific_worker->kicked = true; + specific_worker->kick_state = KICKED; return GRPC_ERROR_NONE; - } else if (specific_worker == g_root_worker) { - specific_worker->kicked = true; + } else if (specific_worker == + (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { + specific_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } else { - specific_worker->kicked = true; + specific_worker->kick_state = KICKED; gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; } @@ -619,9 +734,7 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) {} static grpc_error *kick_poller(void) { - gpr_mu_lock(&g_pollset_mu); - g_timer_kick = true; - gpr_mu_unlock(&g_pollset_mu); + gpr_atm_no_barrier_store(&g_timer_kick, 1); return grpc_wakeup_fd_wakeup(&global_wakeup_fd); } From c9b09e98fd722cd2e798add0acca0a00ef9e7394 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 13:09:10 -0700 Subject: [PATCH 088/177] Fix wakeup condition --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 3355c8344ec..27ed9781060 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -713,7 +713,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } else { return GRPC_ERROR_NONE; } - } else if (specific_worker->kick_state != KICKED) { + } else if (specific_worker->kick_state != UNKICKED) { return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { From 8502ecbd38457ab1bc6de6f4f6c33e6e42b46239 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 14:22:01 -0700 Subject: [PATCH 089/177] Tweak: trigger next poller before exec_ctx flush --- src/core/lib/iomgr/ev_epoll1_linux.c | 26 +++++++++++++++++++------- src/core/lib/iomgr/exec_ctx.c | 5 +++++ src/core/lib/iomgr/exec_ctx.h | 2 ++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 27ed9781060..b1127d38cb6 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -594,14 +594,21 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { - if (worker->kick_state == KICKED_FOR_POLL) { + if (worker_hdl != NULL) *worker_hdl = NULL; + if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { GPR_ASSERT(!pollset->seen_inactive); GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker); if (worker->next != worker) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); + gpr_log(GPR_DEBUG, "Picked sibling worker %p for poller", worker); worker->next->kick_state = KICKED_FOR_POLL; gpr_cv_signal(&worker->next->cv); + if (grpc_exec_ctx_has_work(exec_ctx)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); pollset_neighbourhood *neighbourhood = pollset->neighbourhood; @@ -648,6 +655,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, ((size_t)cur_neighbourhood_idx + 1) % g_num_neighbourhoods; neighbourhood = &g_neighbourhoods[new_neighbourhood_idx]; } while (!found_worker && neighbourhood != pollset->neighbourhood); + grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } } @@ -673,20 +681,18 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->kicked_without_poller = false; return GRPC_ERROR_NONE; } + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { - gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); gpr_mu_unlock(&pollset->mu); append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), err_desc); - grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); - gpr_tls_set(&g_current_thread_pollset, 0); gpr_tls_set(&g_current_thread_worker, 0); - pollset_maybe_finish_shutdown(exec_ctx, pollset); } end_worker(exec_ctx, pollset, &worker, worker_hdl); + gpr_tls_set(&g_current_thread_pollset, 0); return error; } @@ -705,10 +711,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, &g_active_poller)) { root_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); - } else { + } else if (next_worker->kick_state == UNKICKED) { + GPR_ASSERT(next_worker->initialized_cv); next_worker->kick_state = KICKED; gpr_cv_signal(&next_worker->cv); return GRPC_ERROR_NONE; + } else { + return GRPC_ERROR_NONE; } } else { return GRPC_ERROR_NONE; @@ -723,10 +732,13 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, (grpc_pollset_worker *)gpr_atm_no_barrier_load(&g_active_poller)) { specific_worker->kick_state = KICKED; return grpc_wakeup_fd_wakeup(&global_wakeup_fd); - } else { + } else if (specific_worker->initialized_cv) { specific_worker->kick_state = KICKED; gpr_cv_signal(&specific_worker->cv); return GRPC_ERROR_NONE; + } else { + specific_worker->kick_state = KICKED; + return GRPC_ERROR_NONE; } } diff --git a/src/core/lib/iomgr/exec_ctx.c b/src/core/lib/iomgr/exec_ctx.c index 2532a708e7b..318bb2b7133 100644 --- a/src/core/lib/iomgr/exec_ctx.c +++ b/src/core/lib/iomgr/exec_ctx.c @@ -62,6 +62,11 @@ bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) { return true; } +bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx) { + return exec_ctx->active_combiner != NULL || + !grpc_closure_list_empty(exec_ctx->closure_list); +} + bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { bool did_something = 0; GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0); diff --git a/src/core/lib/iomgr/exec_ctx.h b/src/core/lib/iomgr/exec_ctx.h index f99a0fee5fc..759a3ae2d56 100644 --- a/src/core/lib/iomgr/exec_ctx.h +++ b/src/core/lib/iomgr/exec_ctx.h @@ -93,6 +93,8 @@ struct grpc_exec_ctx { extern grpc_closure_scheduler *grpc_schedule_on_exec_ctx; +bool grpc_exec_ctx_has_work(grpc_exec_ctx *exec_ctx); + /** Flush any work that has been enqueued onto this grpc_exec_ctx. * Caller must guarantee that no interfering locks are held. * Returns true if work was performed, false otherwise. */ From f3e40227c39076c7ded7b2217d218f5829830723 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 21:26:35 +0000 Subject: [PATCH 090/177] Fix bad assumption --- src/core/lib/iomgr/ev_epoll1_linux.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index b1127d38cb6..a25168fdb48 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -601,7 +601,6 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (worker->next != worker) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); - gpr_log(GPR_DEBUG, "Picked sibling worker %p for poller", worker); worker->next->kick_state = KICKED_FOR_POLL; gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { @@ -629,9 +628,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (inspect_worker != NULL) { if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { - GPR_ASSERT(inspect_worker->initialized_cv); inspect_worker->kick_state = KICKED_FOR_POLL; - gpr_cv_signal(&inspect_worker->cv); + if (inspect_worker->initialized_cv) gpr_cv_signal(&inspect_worker->cv); } // even if we didn't win the cas, there's a worker, we can stop found_worker = true; From bbf4c7a00a3b81a600821d5a6be94f2021703e04 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 15:12:10 -0700 Subject: [PATCH 091/177] Try to avoid contention --- src/core/lib/iomgr/ev_epoll1_linux.c | 106 +++++++++++++++++---------- 1 file changed, 66 insertions(+), 40 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index a25168fdb48..27f448f08e1 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -314,6 +314,7 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; +static bool *g_neighbour_scan_state; static size_t g_num_neighbourhoods; /* Return true if first in list */ @@ -368,6 +369,8 @@ static grpc_error *pollset_global_init(void) { g_num_neighbourhoods = GPR_MAX(1, gpr_cpu_num_cores()); g_neighbourhoods = gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); + g_neighbour_scan_state = + gpr_malloc(sizeof(*g_neighbour_scan_state) * g_num_neighbourhoods); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_init(&g_neighbourhoods[i].mu); g_neighbourhoods[i].seen_inactive = true; @@ -383,6 +386,7 @@ static void pollset_global_shutdown(void) { gpr_mu_destroy(&g_neighbourhoods[i].mu); } gpr_free(g_neighbourhoods); + gpr_free(g_neighbour_scan_state); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -591,6 +595,42 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, pollset->shutdown_closure == NULL; } +static bool check_neighbourhood_for_available_poller( + pollset_neighbourhood *neighbourhood, grpc_pollset_worker *avoid_worker) { + bool found_worker = false; + do { + grpc_pollset *inspect = neighbourhood->active_root; + if (inspect == NULL) { + break; + } + gpr_mu_lock(&inspect->mu); + GPR_ASSERT(!inspect->seen_inactive); + grpc_pollset_worker *inspect_worker = inspect->root_worker; + if (inspect_worker == avoid_worker) inspect_worker = inspect_worker->next; + if (inspect_worker == avoid_worker) inspect_worker = NULL; + if (inspect_worker != NULL) { + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = KICKED_FOR_POLL; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + } else { + inspect->seen_inactive = true; + move_pollset_to_neighbourhood_list(inspect, &neighbourhood->active_root, + &neighbourhood->inactive_root); + } + gpr_mu_unlock(&inspect->mu); + } while (!found_worker); + if (!found_worker) { + neighbourhood->seen_inactive = true; + } + return found_worker; +} + static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { @@ -610,49 +650,35 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } else { gpr_atm_no_barrier_store(&g_active_poller, 0); - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); + size_t poller_neighbourhood_idx = + (size_t)(pollset->neighbourhood - g_neighbourhoods); bool found_worker = false; - do { - gpr_mu_lock(&neighbourhood->mu); - do { - grpc_pollset *inspect = neighbourhood->active_root; - if (inspect == NULL) { - break; - } - gpr_mu_lock(&inspect->mu); - GPR_ASSERT(!inspect->seen_inactive); - grpc_pollset_worker *inspect_worker = inspect->root_worker; - if (inspect_worker == worker) inspect_worker = worker->next; - if (inspect_worker == worker) inspect_worker = NULL; - if (inspect_worker != NULL) { - if (gpr_atm_no_barrier_cas(&g_active_poller, 0, - (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = KICKED_FOR_POLL; - if (inspect_worker->initialized_cv) gpr_cv_signal(&inspect_worker->cv); - } - // even if we didn't win the cas, there's a worker, we can stop - found_worker = true; - } else { - inspect->seen_inactive = true; - move_pollset_to_neighbourhood_list(inspect, - &neighbourhood->active_root, - &neighbourhood->inactive_root); - } - gpr_mu_unlock(&inspect->mu); - } while (!found_worker); - if (!found_worker) { - neighbourhood->seen_inactive = true; + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + if (gpr_mu_trylock(&neighbourhood->mu)) { + found_worker = + check_neighbourhood_for_available_poller(neighbourhood, worker); + gpr_mu_unlock(&neighbourhood->mu); + g_neighbour_scan_state[i] = true; + } else { + g_neighbour_scan_state[i] = false; } - gpr_mu_unlock(&neighbourhood->mu); - ssize_t cur_neighbourhood_idx = neighbourhood - g_neighbourhoods; - GPR_ASSERT(cur_neighbourhood_idx >= 0); - GPR_ASSERT(g_num_neighbourhoods < INTPTR_MAX); - GPR_ASSERT(cur_neighbourhood_idx < (ssize_t)g_neighbourhoods); - size_t new_neighbourhood_idx = - ((size_t)cur_neighbourhood_idx + 1) % g_num_neighbourhoods; - neighbourhood = &g_neighbourhoods[new_neighbourhood_idx]; - } while (!found_worker && neighbourhood != pollset->neighbourhood); + } + if (!found_worker) { + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + if (g_neighbour_scan_state[i]) continue; + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + gpr_mu_lock(&neighbourhood->mu); + found_worker = + check_neighbourhood_for_available_poller(neighbourhood, worker); + gpr_mu_unlock(&neighbourhood->mu); + } + } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } From b9b2cbfb7177a76afe78501da7ccbaea8fcbfd81 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:12:39 +0000 Subject: [PATCH 092/177] rr assignments --- include/grpc++/server_builder.h | 2 +- src/core/lib/iomgr/ev_epoll1_linux.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index 7ac23349c84..d9de83226bb 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,7 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(1), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(GPR_MAX(1,gpr_cpu_num_cores())), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index a25168fdb48..6f0416d0d3e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -386,9 +386,12 @@ static void pollset_global_shutdown(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + static gpr_atm next_neighbourhood; + gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset->neighbourhood = &g_neighbourhoods[(size_t)gpr_atm_no_barrier_fetch_add(&next_neighbourhood, 1) % g_num_neighbourhoods]; +gpr_log(GPR_DEBUG, "nh=%d", (int)(pollset->neighbourhood - g_neighbourhoods)); pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } From 8affd20a9986d294296b625b35fd9ef42127d437 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:41:40 +0000 Subject: [PATCH 093/177] reduce epoll events to increase parallelism, kill spam --- src/core/lib/iomgr/ev_epoll1_linux.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index f45be3af384..1794af0ac3e 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -395,7 +395,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; pollset->neighbourhood = &g_neighbourhoods[(size_t)gpr_atm_no_barrier_fetch_add(&next_neighbourhood, 1) % g_num_neighbourhoods]; -gpr_log(GPR_DEBUG, "nh=%d", (int)(pollset->neighbourhood - g_neighbourhoods)); pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } @@ -451,7 +450,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 100 +#define MAX_EPOLL_EVENTS 10 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { From a346802846567c15e678a55cff1f957a2c284cc2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:48:00 +0000 Subject: [PATCH 094/177] hmm --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 1794af0ac3e..34d5cd039c3 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -450,7 +450,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 10 +#define MAX_EPOLL_EVENTS 100 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { From b4bfc4ac5d83f6e99a1d2762a990c6ccc3f1e2f3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 22:58:54 +0000 Subject: [PATCH 095/177] hmmmm --- src/core/lib/iomgr/ev_epoll1_linux.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 34d5cd039c3..27f448f08e1 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -390,11 +390,9 @@ static void pollset_global_shutdown(void) { } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { - static gpr_atm next_neighbourhood; - gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[(size_t)gpr_atm_no_barrier_fetch_add(&next_neighbourhood, 1) % g_num_neighbourhoods]; + pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } From 7cb2698abdd599adc67389d7da56d84d29d6a1e5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 23:05:37 +0000 Subject: [PATCH 096/177] Lower epoll ev count --- src/core/lib/iomgr/ev_epoll1_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 27f448f08e1..b9051338b1b 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -448,7 +448,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 100 +#define MAX_EPOLL_EVENTS 10 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { From f4b0fcaa8e046d5ee00dd896c9e674fb249792b2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 16:08:42 -0700 Subject: [PATCH 097/177] Add comment --- src/core/lib/iomgr/ev_epoll1_linux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 27f448f08e1..612cdc4c850 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -563,6 +563,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, // active list pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); + // pollset unlocked: state may change (even worker->kick_state) gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (pollset->seen_inactive) { From 43bf259f4e03eb228602fb46f7325205c077ac0e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 28 Apr 2017 23:21:01 +0000 Subject: [PATCH 098/177] Fix kicking poller --- src/core/lib/iomgr/ev_epoll1_linux.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index b9051338b1b..02fa4023888 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -96,7 +96,7 @@ static void fd_global_shutdown(void); * Pollset Declarations */ -typedef enum { UNKICKED, KICKED, KICKED_FOR_POLL } kick_state; +typedef enum { UNKICKED, KICKED, DESIGNATED_POLLER } kick_state; struct grpc_pollset_worker { kick_state kick_state; @@ -572,7 +572,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (neighbourhood->seen_inactive) { neighbourhood->seen_inactive = false; if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { - worker->kick_state = KICKED_FOR_POLL; + worker->kick_state = DESIGNATED_POLLER; } } } @@ -591,7 +591,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, *now = gpr_now(now->clock_type); } - return worker->kick_state == KICKED_FOR_POLL && + return worker->kick_state == DESIGNATED_POLLER && pollset->shutdown_closure == NULL; } @@ -611,7 +611,7 @@ static bool check_neighbourhood_for_available_poller( if (inspect_worker != NULL) { if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = KICKED_FOR_POLL; + inspect_worker->kick_state = DESIGNATED_POLLER; if (inspect_worker->initialized_cv) { gpr_cv_signal(&inspect_worker->cv); } @@ -641,7 +641,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (worker->next != worker) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); - worker->next->kick_state = KICKED_FOR_POLL; + worker->next->kick_state = DESIGNATED_POLLER; gpr_cv_signal(&worker->next->cv); if (grpc_exec_ctx_has_work(exec_ctx)) { gpr_mu_unlock(&pollset->mu); @@ -746,7 +746,7 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, } else { return GRPC_ERROR_NONE; } - } else if (specific_worker->kick_state != UNKICKED) { + } else if (specific_worker->kick_state == KICKED) { return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { From a4b8eb003ecd69a77f82e6db2b2d825def890de9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 29 Apr 2017 00:13:52 +0000 Subject: [PATCH 099/177] Fix loss of poller bug --- src/core/lib/iomgr/ev_epoll1_linux.c | 39 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 9fb640af6ba..5c158baa773 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -581,6 +581,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } worker_insert(pollset, worker); if (worker->kick_state == UNKICKED) { + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); do { @@ -597,7 +598,7 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, } static bool check_neighbourhood_for_available_poller( - pollset_neighbourhood *neighbourhood, grpc_pollset_worker *avoid_worker) { + pollset_neighbourhood *neighbourhood) { bool found_worker = false; do { grpc_pollset *inspect = neighbourhood->active_root; @@ -607,19 +608,24 @@ static bool check_neighbourhood_for_available_poller( gpr_mu_lock(&inspect->mu); GPR_ASSERT(!inspect->seen_inactive); grpc_pollset_worker *inspect_worker = inspect->root_worker; - if (inspect_worker == avoid_worker) inspect_worker = inspect_worker->next; - if (inspect_worker == avoid_worker) inspect_worker = NULL; if (inspect_worker != NULL) { - if (gpr_atm_no_barrier_cas(&g_active_poller, 0, - (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = DESIGNATED_POLLER; - if (inspect_worker->initialized_cv) { - gpr_cv_signal(&inspect_worker->cv); + do { + if (inspect_worker->kick_state == UNKICKED) { + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = DESIGNATED_POLLER; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + } + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + break; } - } - // even if we didn't win the cas, there's a worker, we can stop - found_worker = true; - } else { + inspect_worker = inspect_worker->next; + } while (inspect_worker != inspect->root_worker); + } + if (!found_worker) { inspect->seen_inactive = true; move_pollset_to_neighbourhood_list(inspect, &neighbourhood->active_root, &neighbourhood->inactive_root); @@ -636,10 +642,10 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl) { if (worker_hdl != NULL) *worker_hdl = NULL; + worker->kick_state = KICKED; if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { GPR_ASSERT(!pollset->seen_inactive); - GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker); - if (worker->next != worker) { + if (worker->next != worker && worker->next->kick_state == UNKICKED) { assert(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); worker->next->kick_state = DESIGNATED_POLLER; @@ -661,7 +667,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; if (gpr_mu_trylock(&neighbourhood->mu)) { found_worker = - check_neighbourhood_for_available_poller(neighbourhood, worker); + check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); g_neighbour_scan_state[i] = true; } else { @@ -676,7 +682,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); found_worker = - check_neighbourhood_for_available_poller(neighbourhood, worker); + check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } } @@ -690,6 +696,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (EMPTIED == worker_remove(pollset, worker)) { pollset_maybe_finish_shutdown(exec_ctx, pollset); } + GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); } /* pollset->po.mu lock must be held by the caller before calling this. From 2acab6e53a9b2704070d8da3451b7b7b89ad090c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 30 Apr 2017 23:06:33 +0000 Subject: [PATCH 100/177] Allow neighbourhood reassignment, add debug --- src/core/lib/iomgr/ev_epoll1_linux.c | 81 ++++++++++++---------------- 1 file changed, 34 insertions(+), 47 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 5c158baa773..1284891ded2 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -109,8 +109,6 @@ struct grpc_pollset_worker { typedef struct pollset_neighbourhood { gpr_mu mu; grpc_pollset *active_root; - grpc_pollset *inactive_root; - bool seen_inactive; char pad[GPR_CACHELINE_SIZE]; } pollset_neighbourhood; @@ -373,7 +371,6 @@ static grpc_error *pollset_global_init(void) { gpr_malloc(sizeof(*g_neighbour_scan_state) * g_num_neighbourhoods); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_init(&g_neighbourhoods[i].mu); - g_neighbourhoods[i].seen_inactive = true; } return GRPC_ERROR_NONE; } @@ -404,9 +401,6 @@ static void pollset_destroy(grpc_pollset *pollset) { if (pollset == pollset->neighbourhood->active_root) { pollset->neighbourhood->active_root = pollset->next == pollset ? NULL : pollset->next; - } else if (pollset == pollset->neighbourhood->inactive_root) { - pollset->neighbourhood->inactive_root = - pollset->next == pollset ? NULL : pollset->next; } gpr_mu_unlock(&pollset->neighbourhood->mu); gpr_mu_destroy(&pollset->mu); @@ -532,25 +526,6 @@ static void verify_neighbourhood_lists(pollset_neighbourhood *neighbourhood) { } #endif -static void move_pollset_to_neighbourhood_list(grpc_pollset *pollset, - grpc_pollset **from_root, - grpc_pollset **to_root) { - // remove from old list - pollset->prev->next = pollset->next; - pollset->next->prev = pollset->prev; - if (*from_root == pollset) { - *from_root = pollset->next == pollset ? NULL : pollset->next; - } - // add to new list - if (*to_root == NULL) { - *to_root = pollset->next = pollset->prev = pollset; - } else { - pollset->next = *to_root; - pollset->prev = pollset->next->prev; - pollset->next->prev = pollset->prev->next = pollset; - } -} - static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { @@ -561,20 +536,29 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (pollset->seen_inactive) { // pollset has been observed to be inactive, we need to move back to the // active list - pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + pollset_neighbourhood *neighbourhood = pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) +retry_lock_neighbourhood: gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (pollset->seen_inactive) { + if (neighbourhood != pollset->neighbourhood) { + gpr_mu_unlock(&neighbourhood->mu); + neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + goto retry_lock_neighbourhood; + } pollset->seen_inactive = false; - move_pollset_to_neighbourhood_list(pollset, &neighbourhood->inactive_root, - &neighbourhood->active_root); - if (neighbourhood->seen_inactive) { - neighbourhood->seen_inactive = false; + if (neighbourhood->active_root == NULL) { + neighbourhood->active_root = pollset->next = pollset->prev = pollset; if (gpr_atm_no_barrier_cas(&g_active_poller, 0, (gpr_atm)worker)) { worker->kick_state = DESIGNATED_POLLER; } + } else { + pollset->next = neighbourhood->active_root; + pollset->prev = pollset->next->prev; + pollset->next->prev = pollset->prev->next = pollset; } } gpr_mu_unlock(&neighbourhood->mu); @@ -627,14 +611,18 @@ static bool check_neighbourhood_for_available_poller( } if (!found_worker) { inspect->seen_inactive = true; - move_pollset_to_neighbourhood_list(inspect, &neighbourhood->active_root, - &neighbourhood->inactive_root); + if (inspect == neighbourhood->active_root) { + if (inspect->next == neighbourhood->active_root) { + neighbourhood->active_root = NULL; + } else { + neighbourhood->active_root = inspect->next; + } + } + inspect->next->prev = inspect->prev; + inspect->prev->next = inspect->next; } gpr_mu_unlock(&inspect->mu); } while (!found_worker); - if (!found_worker) { - neighbourhood->seen_inactive = true; - } return found_worker; } @@ -646,7 +634,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { GPR_ASSERT(!pollset->seen_inactive); if (worker->next != worker && worker->next->kick_state == UNKICKED) { - assert(worker->next->initialized_cv); + GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); worker->next->kick_state = DESIGNATED_POLLER; gpr_cv_signal(&worker->next->cv); @@ -674,17 +662,15 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_neighbour_scan_state[i] = false; } } - if (!found_worker) { - for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - if (g_neighbour_scan_state[i]) continue; - pollset_neighbourhood *neighbourhood = - &g_neighbourhoods[(poller_neighbourhood_idx + i) % - g_num_neighbourhoods]; - gpr_mu_lock(&neighbourhood->mu); - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); - gpr_mu_unlock(&neighbourhood->mu); - } + for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { + if (g_neighbour_scan_state[i]) continue; + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(poller_neighbourhood_idx + i) % + g_num_neighbourhoods]; + gpr_mu_lock(&neighbourhood->mu); + found_worker = + check_neighbourhood_for_available_poller(neighbourhood); + gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); @@ -717,6 +703,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (begin_worker(pollset, &worker, worker_hdl, &now, deadline)) { gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); GPR_ASSERT(!pollset->shutdown_closure); + GPR_ASSERT(!pollset->seen_inactive); gpr_mu_unlock(&pollset->mu); append_error(&error, pollset_epoll(exec_ctx, pollset, now, deadline), err_desc); From ba550da853b9e036ff29c5260c8f17ec4c03fc02 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 14:26:31 +0000 Subject: [PATCH 101/177] Fix several races --- src/core/lib/iomgr/ev_epoll1_linux.c | 87 +++++++++++++++++----------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 1284891ded2..fccccccd093 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -106,6 +106,8 @@ struct grpc_pollset_worker { gpr_cv cv; }; +#define MAX_NEIGHBOURHOODS 1024 + typedef struct pollset_neighbourhood { gpr_mu mu; grpc_pollset *active_root; @@ -121,6 +123,7 @@ struct grpc_pollset { bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ grpc_closure *shutdown_closure; /* Called after after shutdown is complete */ + int begin_refs; grpc_pollset *next; grpc_pollset *prev; @@ -312,7 +315,6 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; -static bool *g_neighbour_scan_state; static size_t g_num_neighbourhoods; /* Return true if first in list */ @@ -352,6 +354,10 @@ static worker_remove_result worker_remove(grpc_pollset *pollset, } } +static size_t choose_neighbourhood(void) { + return (size_t)gpr_cpu_current_cpu() % g_num_neighbourhoods; +} + static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); @@ -364,11 +370,9 @@ static grpc_error *pollset_global_init(void) { if (epoll_ctl(g_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != 0) { return GRPC_OS_ERROR(errno, "epoll_ctl"); } - g_num_neighbourhoods = GPR_MAX(1, gpr_cpu_num_cores()); + g_num_neighbourhoods = GPR_CLAMP(gpr_cpu_num_cores(), 1, MAX_NEIGHBOURHOODS); g_neighbourhoods = gpr_zalloc(sizeof(*g_neighbourhoods) * g_num_neighbourhoods); - g_neighbour_scan_state = - gpr_malloc(sizeof(*g_neighbour_scan_state) * g_num_neighbourhoods); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_init(&g_neighbourhoods[i].mu); } @@ -383,26 +387,27 @@ static void pollset_global_shutdown(void) { gpr_mu_destroy(&g_neighbourhoods[i].mu); } gpr_free(g_neighbourhoods); - gpr_free(g_neighbour_scan_state); } static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; pollset->seen_inactive = true; pollset->next = pollset->prev = pollset; } static void pollset_destroy(grpc_pollset *pollset) { - gpr_mu_lock(&pollset->neighbourhood->mu); - pollset->prev->next = pollset->next; - pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = - pollset->next == pollset ? NULL : pollset->next; - } - gpr_mu_unlock(&pollset->neighbourhood->mu); + if (!pollset->seen_inactive) { + gpr_mu_lock(&pollset->neighbourhood->mu); + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } + gpr_mu_unlock(&pollset->neighbourhood->mu); + } gpr_mu_destroy(&pollset->mu); } @@ -428,7 +433,8 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { static void pollset_maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { - if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL) { + if (pollset->shutdown_closure != NULL && pollset->root_worker == NULL && + pollset->begin_refs == 0) { grpc_closure_sched(exec_ctx, pollset->shutdown_closure, GRPC_ERROR_NONE); pollset->shutdown_closure = NULL; } @@ -532,14 +538,16 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; worker->kick_state = UNKICKED; + pollset->begin_refs++; if (pollset->seen_inactive) { // pollset has been observed to be inactive, we need to move back to the // active list - pollset_neighbourhood *neighbourhood = pollset->neighbourhood = &g_neighbourhoods[gpr_cpu_current_cpu()]; + pollset_neighbourhood *neighbourhood = pollset->neighbourhood = + &g_neighbourhoods[choose_neighbourhood()]; gpr_mu_unlock(&pollset->mu); - // pollset unlocked: state may change (even worker->kick_state) -retry_lock_neighbourhood: + // pollset unlocked: state may change (even worker->kick_state) + retry_lock_neighbourhood: gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (pollset->seen_inactive) { @@ -564,16 +572,18 @@ retry_lock_neighbourhood: gpr_mu_unlock(&neighbourhood->mu); } worker_insert(pollset, worker); + pollset->begin_refs--; if (worker->kick_state == UNKICKED) { GPR_ASSERT(gpr_atm_no_barrier_load(&g_active_poller) != (gpr_atm)worker); worker->initialized_cv = true; gpr_cv_init(&worker->cv); - do { + while (worker->kick_state == UNKICKED && + pollset->shutdown_closure == NULL) { if (gpr_cv_wait(&worker->cv, &pollset->mu, deadline) && worker->kick_state == UNKICKED) { worker->kick_state = KICKED; } - } while (worker->kick_state == UNKICKED); + } *now = gpr_now(now->clock_type); } @@ -594,17 +604,24 @@ static bool check_neighbourhood_for_available_poller( grpc_pollset_worker *inspect_worker = inspect->root_worker; if (inspect_worker != NULL) { do { - if (inspect_worker->kick_state == UNKICKED) { - if (gpr_atm_no_barrier_cas(&g_active_poller, 0, - (gpr_atm)inspect_worker)) { - inspect_worker->kick_state = DESIGNATED_POLLER; - if (inspect_worker->initialized_cv) { - gpr_cv_signal(&inspect_worker->cv); + switch (inspect_worker->kick_state) { + case UNKICKED: + if (gpr_atm_no_barrier_cas(&g_active_poller, 0, + (gpr_atm)inspect_worker)) { + inspect_worker->kick_state = DESIGNATED_POLLER; + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } } - } - // even if we didn't win the cas, there's a worker, we can stop - found_worker = true; - break; + // even if we didn't win the cas, there's a worker, we can stop + found_worker = true; + break; + case KICKED: + break; + case DESIGNATED_POLLER: + found_worker = true; // ok, so someone else found the worker, but + // we'll accept that + break; } inspect_worker = inspect_worker->next; } while (inspect_worker != inspect->root_worker); @@ -649,6 +666,7 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, size_t poller_neighbourhood_idx = (size_t)(pollset->neighbourhood - g_neighbourhoods); bool found_worker = false; + bool scan_state[MAX_NEIGHBOURHOODS]; for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { pollset_neighbourhood *neighbourhood = &g_neighbourhoods[(poller_neighbourhood_idx + i) % @@ -657,19 +675,18 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, found_worker = check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); - g_neighbour_scan_state[i] = true; + scan_state[i] = true; } else { - g_neighbour_scan_state[i] = false; + scan_state[i] = false; } } for (size_t i = 0; !found_worker && i < g_num_neighbourhoods; i++) { - if (g_neighbour_scan_state[i]) continue; + if (scan_state[i]) continue; pollset_neighbourhood *neighbourhood = &g_neighbourhoods[(poller_neighbourhood_idx + i) % g_num_neighbourhoods]; gpr_mu_lock(&neighbourhood->mu); - found_worker = - check_neighbourhood_for_available_poller(neighbourhood); + found_worker = check_neighbourhood_for_available_poller(neighbourhood); gpr_mu_unlock(&neighbourhood->mu); } grpc_exec_ctx_flush(exec_ctx); From e00d7335de79fb52ecba1756a991829feb6c1c7d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 15:43:51 +0000 Subject: [PATCH 102/177] Refine neighbourhood reassignment --- src/core/lib/iomgr/ev_epoll1_linux.c | 48 ++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index fccccccd093..99e4441a6c7 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -117,6 +117,7 @@ typedef struct pollset_neighbourhood { struct grpc_pollset { gpr_mu mu; pollset_neighbourhood *neighbourhood; + bool reassigning_neighbourhood; grpc_pollset_worker *root_worker; bool kicked_without_poller; bool seen_inactive; @@ -394,20 +395,33 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { *mu = &pollset->mu; pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; pollset->seen_inactive = true; - pollset->next = pollset->prev = pollset; } static void pollset_destroy(grpc_pollset *pollset) { + gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { - gpr_mu_lock(&pollset->neighbourhood->mu); - pollset->prev->next = pollset->next; - pollset->next->prev = pollset->prev; - if (pollset == pollset->neighbourhood->active_root) { - pollset->neighbourhood->active_root = - pollset->next == pollset ? NULL : pollset->next; + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); +retry_lock_neighbourhood: + gpr_mu_lock(&neighbourhood->mu); + gpr_mu_lock(&pollset->mu); + if (!pollset->seen_inactive) { + if (pollset->neighbourhood != neighbourhood) { + gpr_mu_unlock(&neighbourhood->mu); + neighbourhood = pollset->neighbourhood; + gpr_mu_unlock(&pollset->mu); + goto retry_lock_neighbourhood; + } + pollset->prev->next = pollset->next; + pollset->next->prev = pollset->prev; + if (pollset == pollset->neighbourhood->active_root) { + pollset->neighbourhood->active_root = + pollset->next == pollset ? NULL : pollset->next; + } } gpr_mu_unlock(&pollset->neighbourhood->mu); } + gpr_mu_unlock(&pollset->mu); gpr_mu_destroy(&pollset->mu); } @@ -543,8 +557,13 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (pollset->seen_inactive) { // pollset has been observed to be inactive, we need to move back to the // active list - pollset_neighbourhood *neighbourhood = pollset->neighbourhood = - &g_neighbourhoods[choose_neighbourhood()]; + bool is_reassigning = false; + if (!pollset->reassigning_neighbourhood) { + is_reassigning = true; + pollset->reassigning_neighbourhood = true; + pollset->neighbourhood = &g_neighbourhoods[choose_neighbourhood()]; + } + pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); // pollset unlocked: state may change (even worker->kick_state) retry_lock_neighbourhood: @@ -569,6 +588,10 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, pollset->next->prev = pollset->prev->next = pollset; } } + if (is_reassigning) { + GPR_ASSERT(pollset->reassigning_neighbourhood); + pollset->reassigning_neighbourhood = false; + } gpr_mu_unlock(&neighbourhood->mu); } worker_insert(pollset, worker); @@ -629,14 +652,11 @@ static bool check_neighbourhood_for_available_poller( if (!found_worker) { inspect->seen_inactive = true; if (inspect == neighbourhood->active_root) { - if (inspect->next == neighbourhood->active_root) { - neighbourhood->active_root = NULL; - } else { - neighbourhood->active_root = inspect->next; - } + neighbourhood->active_root = inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; inspect->prev->next = inspect->next; + inspect->next = inspect->prev = NULL; } gpr_mu_unlock(&inspect->mu); } while (!found_worker); From e45a277b56273ae63dca8a0bb65e0337c15f7e37 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 15:56:20 +0000 Subject: [PATCH 103/177] Remove bogus assert --- src/core/lib/iomgr/ev_epoll1_linux.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 99e4441a6c7..8946d326974 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -669,7 +669,6 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (worker_hdl != NULL) *worker_hdl = NULL; worker->kick_state = KICKED; if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { - GPR_ASSERT(!pollset->seen_inactive); if (worker->next != worker && worker->next->kick_state == UNKICKED) { GPR_ASSERT(worker->next->initialized_cv); gpr_atm_no_barrier_store(&g_active_poller, (gpr_atm)worker->next); From a95bacf7dba3d5ffc8968729525a4cd5ff0ff88c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 12:51:24 -0700 Subject: [PATCH 104/177] clang-format, revert parameter --- include/grpc++/server_builder.h | 5 ++++- src/core/lib/iomgr/ev_epoll1_linux.c | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h index d9de83226bb..5a8577659ab 100644 --- a/include/grpc++/server_builder.h +++ b/include/grpc++/server_builder.h @@ -195,7 +195,10 @@ class ServerBuilder { struct SyncServerSettings { SyncServerSettings() - : num_cqs(GPR_MAX(1,gpr_cpu_num_cores())), min_pollers(1), max_pollers(2), cq_timeout_msec(10000) {} + : num_cqs(GPR_MAX(1, gpr_cpu_num_cores())), + min_pollers(1), + max_pollers(2), + cq_timeout_msec(10000) {} // Number of server completion queues to create to listen to incoming RPCs. int num_cqs; diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8946d326974..29254e6324a 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -402,7 +402,7 @@ static void pollset_destroy(grpc_pollset *pollset) { if (!pollset->seen_inactive) { pollset_neighbourhood *neighbourhood = pollset->neighbourhood; gpr_mu_unlock(&pollset->mu); -retry_lock_neighbourhood: + retry_lock_neighbourhood: gpr_mu_lock(&neighbourhood->mu); gpr_mu_lock(&pollset->mu); if (!pollset->seen_inactive) { @@ -462,7 +462,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_maybe_finish_shutdown(exec_ctx, pollset); } -#define MAX_EPOLL_EVENTS 10 +#define MAX_EPOLL_EVENTS 100 static int poll_deadline_to_millis_timeout(gpr_timespec deadline, gpr_timespec now) { @@ -652,7 +652,8 @@ static bool check_neighbourhood_for_available_poller( if (!found_worker) { inspect->seen_inactive = true; if (inspect == neighbourhood->active_root) { - neighbourhood->active_root = inspect->next == inspect ? NULL : inspect->next; + neighbourhood->active_root = + inspect->next == inspect ? NULL : inspect->next; } inspect->next->prev = inspect->prev; inspect->prev->next = inspect->next; From 50da5ec21d3d8be5e76b9809242821f9e5badba1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 13:51:14 -0700 Subject: [PATCH 105/177] Add workqueue --- src/core/lib/iomgr/ev_epoll1_linux.c | 84 +++++++++++++++++++++------- src/core/lib/support/mpscq.c | 13 ++++- src/core/lib/support/mpscq.h | 4 ++ 3 files changed, 80 insertions(+), 21 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 29254e6324a..cce52b2d944 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -104,6 +104,7 @@ struct grpc_pollset_worker { grpc_pollset_worker *next; grpc_pollset_worker *prev; gpr_cv cv; + grpc_closure_list schedule_on_end_work; }; #define MAX_NEIGHBOURHOODS 1024 @@ -288,7 +289,7 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, } static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - return NULL; /* TODO(ctiller): add a global workqueue */ + return (grpc_workqueue *)0xb0b51ed; } static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, @@ -317,6 +318,7 @@ GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; static size_t g_num_neighbourhoods; +static gpr_mpscq g_workqueue_items; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -365,6 +367,7 @@ static grpc_error *pollset_global_init(void) { gpr_atm_no_barrier_store(&g_active_poller, 0); global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); + gpr_mpscq_init(&g_workqueue_items); if (err != GRPC_ERROR_NONE) return err; struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &global_wakeup_fd}; @@ -383,6 +386,7 @@ static grpc_error *pollset_global_init(void) { static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); + gpr_mpscq_destroy(&g_workqueue_items); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_destroy(&g_neighbourhoods[i].mu); @@ -528,30 +532,13 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } -#if 0 -static void verify_all_entries_in_neighbourhood_list( - grpc_pollset *root, bool should_be_seen_inactive) { - if (root == NULL) return; - grpc_pollset *p = root; - do { - GPR_ASSERT(p->seen_inactive == should_be_seen_inactive); - p = p->next; - } while (p != root); -} - -static void verify_neighbourhood_lists(pollset_neighbourhood *neighbourhood) { - // assumes neighbourhood->mu locked - verify_all_entries_in_neighbourhood_list(neighbourhood->active_root, false); - verify_all_entries_in_neighbourhood_list(neighbourhood->inactive_root, true); -} -#endif - static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, grpc_pollset_worker **worker_hdl, gpr_timespec *now, gpr_timespec deadline) { if (worker_hdl != NULL) *worker_hdl = worker; worker->initialized_cv = false; worker->kick_state = UNKICKED; + worker->schedule_on_end_work = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; pollset->begin_refs++; if (pollset->seen_inactive) { @@ -669,6 +656,8 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl) { if (worker_hdl != NULL) *worker_hdl = NULL; worker->kick_state = KICKED; + grpc_closure_list_move(&worker->schedule_on_end_work, + &exec_ctx->closure_list); if (gpr_atm_no_barrier_load(&g_active_poller) == (gpr_atm)worker) { if (worker->next != worker && worker->next->kick_state == UNKICKED) { GPR_ASSERT(worker->next->initialized_cv); @@ -712,6 +701,10 @@ static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } + } else if (grpc_exec_ctx_has_work(exec_ctx)) { + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); } if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); @@ -828,8 +821,59 @@ static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {} #endif +static void wq_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + // find a neighbourhood to wakeup + bool scheduled = false; + size_t initial_neighbourhood = choose_neighbourhood(); + for (size_t i = 0; !scheduled && i < g_num_neighbourhoods; i++) { + pollset_neighbourhood *neighbourhood = + &g_neighbourhoods[(initial_neighbourhood + i) % g_num_neighbourhoods]; + if (gpr_mu_trylock(&neighbourhood->mu)) { + if (neighbourhood->active_root != NULL) { + grpc_pollset *inspect = neighbourhood->active_root; + do { + if (gpr_mu_trylock(&inspect->mu)) { + if (inspect->root_worker != NULL) { + grpc_pollset_worker *inspect_worker = inspect->root_worker; + do { + if (inspect_worker->kick_state == UNKICKED) { + inspect_worker->kick_state = KICKED; + grpc_closure_list_append( + &inspect_worker->schedule_on_end_work, closure, error); + if (inspect_worker->initialized_cv) { + gpr_cv_signal(&inspect_worker->cv); + } + scheduled = true; + } + inspect_worker = inspect_worker->next; + } while (!scheduled && inspect_worker != inspect->root_worker); + } + gpr_mu_unlock(&inspect->mu); + } + inspect = inspect->next; + } while (!scheduled && inspect != neighbourhood->active_root); + } + gpr_mu_unlock(&neighbourhood->mu); + } + } + if (!scheduled) { + closure->error_data.error = error; + gpr_mpscq_push(&g_workqueue_items, &closure->next_data.atm_next); + GRPC_LOG_IF_ERROR("workqueue_scheduler", + grpc_wakeup_fd_wakeup(&global_wakeup_fd)); + } +} + +static const grpc_closure_scheduler_vtable + singleton_workqueue_scheduler_vtable = {wq_sched, wq_sched, + "epoll1_workqueue"}; + +static grpc_closure_scheduler singleton_workqueue_scheduler = { + &singleton_workqueue_scheduler_vtable}; + static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - return grpc_schedule_on_exec_ctx; + return &singleton_workqueue_scheduler; } /******************************************************************************* diff --git a/src/core/lib/support/mpscq.c b/src/core/lib/support/mpscq.c index 5b9323275aa..1015cc67765 100644 --- a/src/core/lib/support/mpscq.c +++ b/src/core/lib/support/mpscq.c @@ -54,21 +54,31 @@ void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) { } gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) { + bool empty; + return gpr_mpscq_pop_and_check_end(q, &empty); +} + +gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty) { gpr_mpscq_node *tail = q->tail; gpr_mpscq_node *next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next); if (tail == &q->stub) { // indicates the list is actually (ephemerally) empty - if (next == NULL) return NULL; + if (next == NULL) { + *empty = true; + return NULL; + } q->tail = next; tail = next; next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next); } if (next != NULL) { + *empty = false; q->tail = next; return tail; } gpr_mpscq_node *head = (gpr_mpscq_node *)gpr_atm_acq_load(&q->head); if (tail != head) { + *empty = false; // indicates a retry is in order: we're still adding return NULL; } @@ -79,5 +89,6 @@ gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) { return tail; } // indicates a retry is in order: we're still adding + *empty = false; return NULL; } diff --git a/src/core/lib/support/mpscq.h b/src/core/lib/support/mpscq.h index 977a1179529..24c89f90c9d 100644 --- a/src/core/lib/support/mpscq.h +++ b/src/core/lib/support/mpscq.h @@ -35,6 +35,7 @@ #define GRPC_CORE_LIB_SUPPORT_MPSCQ_H #include +#include #include // Multiple-producer single-consumer lock free queue, based upon the @@ -62,4 +63,7 @@ void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n); // the queue is empty!!) gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q); +// Pop a node; sets *empty to true if the queue is empty, or false if it is not +gpr_mpscq_node *gpr_mpscq_pop_and_check_end(gpr_mpscq *q, bool *empty); + #endif /* GRPC_CORE_LIB_SUPPORT_MPSCQ_H */ From 67e229e5c124efa826ef15e8a7e16634786fabeb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 20:57:59 +0000 Subject: [PATCH 106/177] Fix wakeup path --- src/core/lib/iomgr/ev_epoll1_linux.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index cce52b2d944..8766e4a1aa2 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -318,7 +318,8 @@ GPR_TLS_DECL(g_current_thread_worker); static gpr_atm g_active_poller; static pollset_neighbourhood *g_neighbourhoods; static size_t g_num_neighbourhoods; -static gpr_mpscq g_workqueue_items; +static gpr_mu g_wq_mu; +static grpc_closure_list g_wq_items; /* Return true if first in list */ static bool worker_insert(grpc_pollset *pollset, grpc_pollset_worker *worker) { @@ -367,7 +368,8 @@ static grpc_error *pollset_global_init(void) { gpr_atm_no_barrier_store(&g_active_poller, 0); global_wakeup_fd.read_fd = -1; grpc_error *err = grpc_wakeup_fd_init(&global_wakeup_fd); - gpr_mpscq_init(&g_workqueue_items); + gpr_mu_init(&g_wq_mu); + g_wq_items = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; if (err != GRPC_ERROR_NONE) return err; struct epoll_event ev = {.events = (uint32_t)(EPOLLIN | EPOLLET), .data.ptr = &global_wakeup_fd}; @@ -386,7 +388,7 @@ static grpc_error *pollset_global_init(void) { static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); - gpr_mpscq_destroy(&g_workqueue_items); + gpr_mu_destroy(&g_wq_mu); if (global_wakeup_fd.read_fd != -1) grpc_wakeup_fd_destroy(&global_wakeup_fd); for (size_t i = 0; i < g_num_neighbourhoods; i++) { gpr_mu_destroy(&g_neighbourhoods[i].mu); @@ -513,6 +515,9 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (gpr_atm_no_barrier_cas(&g_timer_kick, 1, 0)) { grpc_timer_consume_kick(); } + gpr_mu_lock(&g_wq_mu); + grpc_closure_list_move(&g_wq_items, &exec_ctx->closure_list); + gpr_mu_unlock(&g_wq_mu); append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); } else { @@ -858,8 +863,9 @@ static void wq_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure, } } if (!scheduled) { - closure->error_data.error = error; - gpr_mpscq_push(&g_workqueue_items, &closure->next_data.atm_next); + gpr_mu_lock(&g_wq_mu); + grpc_closure_list_append(&g_wq_items, closure, error); + gpr_mu_unlock(&g_wq_mu); GRPC_LOG_IF_ERROR("workqueue_scheduler", grpc_wakeup_fd_wakeup(&global_wakeup_fd)); } From 5c53d1bd0af5b4d2cd79255b7d65329a2c0b65c9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 1 May 2017 22:41:22 +0000 Subject: [PATCH 107/177] Fix merge --- tools/run_tests/run_tests.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ab9efd58d27..2da2ffa6726 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -72,11 +72,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { _POLLING_STRATEGIES = { -<<<<<<< HEAD - 'linux': ['epollex', 'epoll', 'poll', 'poll-cv'] -======= - 'linux': ['epoll1', 'epollsig', 'poll', 'poll-cv'] ->>>>>>> 50da5ec21d3d8be5e76b9809242821f9e5badba1 + 'linux': ['epollex', 'epoll1', 'epollsig', 'poll', 'poll-cv'] } From c3571791a5e20ed82cc2efebd21d48f898f13eba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 2 May 2017 12:33:38 -0700 Subject: [PATCH 108/177] Isolate timer checking in its own thread --- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + src/core/lib/iomgr/ev_epoll_linux.c | 25 +-- src/core/lib/iomgr/ev_poll_posix.c | 31 +--- src/core/lib/iomgr/ev_posix.c | 2 - src/core/lib/iomgr/ev_posix.h | 2 - src/core/lib/iomgr/iomgr.c | 3 + src/core/lib/iomgr/pollset_windows.c | 2 - src/core/lib/iomgr/timer_manager.c | 166 ++++++++++++++++++ src/core/lib/iomgr/timer_manager.h | 43 +++++ src/core/lib/surface/completion_queue.c | 67 +++---- src/python/grpcio/grpc_core_dependencies.py | 1 + test/cpp/qps/client_async.cc | 48 ++--- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util.vcxproj.filters | 6 + .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + 32 files changed, 340 insertions(+), 127 deletions(-) create mode 100644 src/core/lib/iomgr/timer_manager.c create mode 100644 src/core/lib/iomgr/timer_manager.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 325344a81c9..b52a42f7e00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -984,6 +984,7 @@ add_library(grpc src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -1310,6 +1311,7 @@ add_library(grpc_cronet src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -1621,6 +1623,7 @@ add_library(grpc_test_util src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -1877,6 +1880,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -2296,6 +2300,7 @@ add_library(grpc++ src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -2621,6 +2626,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c @@ -3390,6 +3396,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/time_averaged_stats.c src/core/lib/iomgr/timer_generic.c src/core/lib/iomgr/timer_heap.c + src/core/lib/iomgr/timer_manager.c src/core/lib/iomgr/timer_uv.c src/core/lib/iomgr/udp_server.c src/core/lib/iomgr/unix_sockets_posix.c diff --git a/Makefile b/Makefile index 4597b6fe974..d9de26f5469 100644 --- a/Makefile +++ b/Makefile @@ -2967,6 +2967,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -3291,6 +3292,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -3601,6 +3603,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -3829,6 +3832,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -4225,6 +4229,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -4558,6 +4563,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ @@ -5317,6 +5323,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ diff --git a/binding.gyp b/binding.gyp index 582c61282f9..9434ab6d32f 100644 --- a/binding.gyp +++ b/binding.gyp @@ -718,6 +718,7 @@ 'src/core/lib/iomgr/time_averaged_stats.c', 'src/core/lib/iomgr/timer_generic.c', 'src/core/lib/iomgr/timer_heap.c', + 'src/core/lib/iomgr/timer_manager.c', 'src/core/lib/iomgr/timer_uv.c', 'src/core/lib/iomgr/udp_server.c', 'src/core/lib/iomgr/unix_sockets_posix.c', diff --git a/build.yaml b/build.yaml index 4cfa75cae14..9453c1a2b38 100644 --- a/build.yaml +++ b/build.yaml @@ -238,6 +238,7 @@ filegroups: - src/core/lib/iomgr/timer.h - src/core/lib/iomgr/timer_generic.h - src/core/lib/iomgr/timer_heap.h + - src/core/lib/iomgr/timer_manager.h - src/core/lib/iomgr/timer_uv.h - src/core/lib/iomgr/udp_server.h - src/core/lib/iomgr/unix_sockets_posix.h @@ -350,6 +351,7 @@ filegroups: - src/core/lib/iomgr/time_averaged_stats.c - src/core/lib/iomgr/timer_generic.c - src/core/lib/iomgr/timer_heap.c + - src/core/lib/iomgr/timer_manager.c - src/core/lib/iomgr/timer_uv.c - src/core/lib/iomgr/udp_server.c - src/core/lib/iomgr/unix_sockets_posix.c diff --git a/config.m4 b/config.m4 index bbd667c9ec4..36c04cf07b6 100644 --- a/config.m4 +++ b/config.m4 @@ -154,6 +154,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/time_averaged_stats.c \ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_heap.c \ + src/core/lib/iomgr/timer_manager.c \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/udp_server.c \ src/core/lib/iomgr/unix_sockets_posix.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 07755ac7279..120463f40c6 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -320,6 +320,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/timer.h', 'src/core/lib/iomgr/timer_generic.h', 'src/core/lib/iomgr/timer_heap.h', + 'src/core/lib/iomgr/timer_manager.h', 'src/core/lib/iomgr/timer_uv.h', 'src/core/lib/iomgr/udp_server.h', 'src/core/lib/iomgr/unix_sockets_posix.h', @@ -531,6 +532,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/time_averaged_stats.c', 'src/core/lib/iomgr/timer_generic.c', 'src/core/lib/iomgr/timer_heap.c', + 'src/core/lib/iomgr/timer_manager.c', 'src/core/lib/iomgr/timer_uv.c', 'src/core/lib/iomgr/udp_server.c', 'src/core/lib/iomgr/unix_sockets_posix.c', @@ -781,6 +783,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/timer.h', 'src/core/lib/iomgr/timer_generic.h', 'src/core/lib/iomgr/timer_heap.h', + 'src/core/lib/iomgr/timer_manager.h', 'src/core/lib/iomgr/timer_uv.h', 'src/core/lib/iomgr/udp_server.h', 'src/core/lib/iomgr/unix_sockets_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index 1cd6d66335e..7841f271027 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -236,6 +236,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/timer.h ) s.files += %w( src/core/lib/iomgr/timer_generic.h ) s.files += %w( src/core/lib/iomgr/timer_heap.h ) + s.files += %w( src/core/lib/iomgr/timer_manager.h ) s.files += %w( src/core/lib/iomgr/timer_uv.h ) s.files += %w( src/core/lib/iomgr/udp_server.h ) s.files += %w( src/core/lib/iomgr/unix_sockets_posix.h ) @@ -447,6 +448,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/time_averaged_stats.c ) s.files += %w( src/core/lib/iomgr/timer_generic.c ) s.files += %w( src/core/lib/iomgr/timer_heap.c ) + s.files += %w( src/core/lib/iomgr/timer_manager.c ) s.files += %w( src/core/lib/iomgr/timer_uv.c ) s.files += %w( src/core/lib/iomgr/udp_server.c ) s.files += %w( src/core/lib/iomgr/unix_sockets_posix.c ) diff --git a/package.xml b/package.xml index e7d67eca180..3f7ddb9e49e 100644 --- a/package.xml +++ b/package.xml @@ -245,6 +245,7 @@ + @@ -456,6 +457,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index e603a755937..735c37be954 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -76,11 +76,6 @@ static int grpc_polling_trace = 0; /* Disabled by default */ static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - /* Implements the function defined in grpc_posix.h. This function might be * called before even calling grpc_init() to set either a different signal to * use. If signum == -1, then the use of signals is disabled */ @@ -454,8 +449,8 @@ static void polling_island_add_wakeup_fd_locked(polling_island *pi, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), - errno, strerror(errno)); + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -558,7 +553,6 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, goto done; } - polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); if (initial_fd != NULL) { @@ -1116,11 +1110,10 @@ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); poller_kick_init(); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } @@ -1226,10 +1219,6 @@ static grpc_error *pollset_kick(grpc_pollset *p, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->po.mu); *mu = &pollset->po.mu; @@ -1453,11 +1442,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, for (int i = 0; i < ep_rv; ++i) { void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { + if (data_ptr == &pi->workqueue_wakeup_fd) { append_error(error, grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), err_desc); @@ -1897,8 +1882,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index 9834cdd1979..3ef24be5697 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -122,8 +122,6 @@ struct grpc_fd { grpc_pollset *read_notifier_pollset; }; -static grpc_wakeup_fd global_wakeup_fd; - /* Begin polling on an fd. Registers that the given pollset is interested in this fd - so that if read or writability interest changes, the pollset can be kicked to pick up that @@ -784,19 +782,14 @@ static grpc_error *pollset_kick(grpc_pollset *p, static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_poller); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_poller); gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - /* main interface */ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { @@ -952,13 +945,10 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } fd_count = 0; - pfd_count = 2; - pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd); + pfd_count = 1; + pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd); pfds[0].events = POLLIN; pfds[0].revents = 0; - pfds[1].fd = GRPC_WAKEUP_FD_GET_READ_FD(&worker.wakeup_fd->fd); - pfds[1].events = POLLIN; - pfds[1].revents = 0; for (i = 0; i < pollset->fd_count; i++) { if (fd_is_orphaned(pollset->fds[i])) { GRPC_FD_UNREF(pollset->fds[i], "multipoller"); @@ -974,7 +964,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset->fd_count = fd_count; gpr_mu_unlock(&pollset->mu); - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { grpc_fd *fd = watchers[i].fd; pfds[i].events = (short)fd_begin_poll(fd, pollset, &worker, POLLIN, POLLOUT, &watchers[i]); @@ -992,7 +982,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, work_combine_error(&error, GRPC_OS_ERROR(errno, "poll")); } - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == NULL) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } else { @@ -1002,20 +992,15 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } } } else if (r == 0) { - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } } else { if (pfds[0].revents & POLLIN_CHECK) { - grpc_timer_consume_kick(); - work_combine_error(&error, - grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd)); - } - if (pfds[1].revents & POLLIN_CHECK) { work_combine_error( &error, grpc_wakeup_fd_consume_wakeup(&worker.wakeup_fd->fd)); } - for (i = 2; i < pfd_count; i++) { + for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == NULL) { fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL); } else { @@ -1560,8 +1545,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de83..41464c137a9 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -260,8 +260,6 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, g_event_engine->pollset_set_del_fd(exec_ctx, pollset_set, fd); } -grpc_error *grpc_kick_poller(void) { return g_event_engine->kick_poller(); } - #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG grpc_workqueue *grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, const char *reason) { diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index becc4d359e5..a77720e61fd 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -93,8 +93,6 @@ typedef struct grpc_event_engine_vtable { void (*pollset_set_del_fd)(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pollset_set, grpc_fd *fd); - grpc_error *(*kick_poller)(void); - void (*shutdown_engine)(void); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 001e5284099..445f7aa422d 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -47,6 +47,7 @@ #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/network_status_tracker.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/support/env.h" #include "src/core/lib/support/string.h" @@ -65,6 +66,7 @@ void grpc_iomgr_init(void) { g_root_object.name = "root"; grpc_network_status_init(); grpc_iomgr_platform_init(); + grpc_timer_manager_init(); } static size_t count_objects(void) { @@ -88,6 +90,7 @@ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx) { gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_seconds(10, GPR_TIMESPAN)); gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME); + grpc_timer_manager_shutdown(); grpc_iomgr_platform_flush(); gpr_mu_lock(&g_mu); diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 04c6b717475..bea4232273a 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -227,6 +227,4 @@ grpc_error *grpc_pollset_kick(grpc_pollset *p, return GRPC_ERROR_NONE; } -void grpc_kick_poller(void) { grpc_iocp_kick(); } - #endif /* GRPC_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c new file mode 100644 index 00000000000..00e868de016 --- /dev/null +++ b/src/core/lib/iomgr/timer_manager.c @@ -0,0 +1,166 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/timer_manager.h" + +#include +#include +#include + +#include "src/core/lib/iomgr/timer.h" + +typedef struct completed_thread { + gpr_thd_id t; + struct completed_thread *next; +} completed_thread; + +static gpr_mu g_mu; +static gpr_cv g_cv_wait; +static gpr_cv g_cv_shutdown; +static int g_thread_count; +static int g_waiter_count; +static bool g_shutdown; +static completed_thread *g_completed_threads; +static bool g_kicked; + +#define MAX_WAITERS 3 + +static void timer_thread(void *unused); + +static void gc_completed_threads(void) { + if (g_completed_threads != NULL) { + completed_thread *to_gc = g_completed_threads; + g_completed_threads = NULL; + gpr_mu_unlock(&g_mu); + while (to_gc != NULL) { + gpr_thd_join(to_gc->t); + completed_thread *next = to_gc->next; + gpr_free(to_gc); + to_gc = next; + } + gpr_mu_lock(&g_mu); + } +} + +static void start_timer_thread_and_unlock(void) { + ++g_waiter_count; + ++g_thread_count; + gpr_mu_unlock(&g_mu); + gpr_log(GPR_DEBUG, "Spawn timer thread"); + gpr_thd_id thd; + gpr_thd_options opt = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&opt); + gpr_thd_new(&thd, timer_thread, NULL, &opt); +} + +static void timer_thread(void *unused) { + grpc_exec_ctx exec_ctx = + GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + for (;;) { + gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + if (grpc_timer_check(&exec_ctx, now, &next)) { + gpr_mu_lock(&g_mu); + --g_waiter_count; + bool start_thread = g_waiter_count == 0; + if (start_thread && !g_shutdown) { + start_timer_thread_and_unlock(); + } else { + gpr_mu_unlock(&g_mu); + } + grpc_exec_ctx_flush(&exec_ctx); + gpr_mu_lock(&g_mu); + gc_completed_threads(); + ++g_waiter_count; + gpr_mu_unlock(&g_mu); + } else { + gpr_mu_lock(&g_mu); + if (g_shutdown) break; + if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) { + if (g_kicked) { + grpc_timer_consume_kick(); + g_kicked = false; + } else if (g_waiter_count > MAX_WAITERS) { + break; + } + } + gpr_mu_unlock(&g_mu); + } + } + --g_waiter_count; + --g_thread_count; + if (0 == g_thread_count) { + gpr_cv_signal(&g_cv_shutdown); + } + completed_thread *ct = gpr_malloc(sizeof(*ct)); + ct->t = gpr_thd_currentid(); + ct->next = g_completed_threads; + g_completed_threads = ct; + gpr_mu_unlock(&g_mu); + gpr_log(GPR_DEBUG, "End timer thread"); +} + +void grpc_timer_manager_init(void) { + gpr_mu_init(&g_mu); + gpr_cv_init(&g_cv_wait); + gpr_cv_init(&g_cv_shutdown); + g_thread_count = 0; + g_waiter_count = 0; + g_shutdown = false; + g_completed_threads = NULL; + + gpr_mu_lock(&g_mu); + start_timer_thread_and_unlock(); +} + +void grpc_timer_manager_shutdown(void) { + gpr_mu_lock(&g_mu); + g_shutdown = true; + gpr_cv_broadcast(&g_cv_wait); + while (g_thread_count > 0) { + gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + gc_completed_threads(); + } + gpr_mu_unlock(&g_mu); + + gpr_mu_destroy(&g_mu); + gpr_cv_destroy(&g_cv_wait); + gpr_cv_destroy(&g_cv_shutdown); +} + +void grpc_kick_poller(void) { + gpr_mu_lock(&g_mu); + g_kicked = true; + gpr_cv_signal(&g_cv_wait); + gpr_mu_unlock(&g_mu); +} diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h new file mode 100644 index 00000000000..a24c9da328a --- /dev/null +++ b/src/core/lib/iomgr/timer_manager.h @@ -0,0 +1,43 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H +#define GRPC_CORE_IOMGR_TIMER_MANAGER_H + +/* Timer Manager tries to keep one thread waiting for the next timeout at all + times */ + +void grpc_timer_manager_init(void); +void grpc_timer_manager_shutdown(void); + +#endif diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index eae3f103b12..048564e32f9 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -580,31 +580,18 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, dump_pending_tags(cc); break; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - gpr_timespec iteration_deadline = deadline; - if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { - GPR_TIMER_MARK("alarm_triggered", 0); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + NULL, now, deadline); + if (err != GRPC_ERROR_NONE) { gpr_mu_unlock(cc->mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - continue; - } else { - grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), - NULL, now, iteration_deadline); - if (err != GRPC_ERROR_NONE) { - gpr_mu_unlock(cc->mu); - const char *msg = grpc_error_string(err); - gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); + const char *msg = grpc_error_string(err); + gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); - GRPC_ERROR_UNREF(err); - memset(&ret, 0, sizeof(ret)); - ret.type = GRPC_QUEUE_TIMEOUT; - dump_pending_tags(cc); - break; - } + GRPC_ERROR_UNREF(err); + memset(&ret, 0, sizeof(ret)); + ret.type = GRPC_QUEUE_TIMEOUT; + dump_pending_tags(cc); + break; } is_finished_arg.first_loop = false; } @@ -773,31 +760,19 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, dump_pending_tags(cc); break; } - /* Check alarms - these are a global resource so we just ping - each time through on every pollset. - May update deadline to ensure timely wakeups. - TODO(ctiller): can this work be localized? */ - gpr_timespec iteration_deadline = deadline; - if (grpc_timer_check(&exec_ctx, now, &iteration_deadline)) { - GPR_TIMER_MARK("alarm_triggered", 0); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + &worker, now, deadline); + if (err != GRPC_ERROR_NONE) { + del_plucker(cc, tag, &worker); gpr_mu_unlock(cc->mu); - grpc_exec_ctx_flush(&exec_ctx); - gpr_mu_lock(cc->mu); - } else { - grpc_error *err = cc->poller_vtable->work( - &exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); - if (err != GRPC_ERROR_NONE) { - del_plucker(cc, tag, &worker); - gpr_mu_unlock(cc->mu); - const char *msg = grpc_error_string(err); - gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); + const char *msg = grpc_error_string(err); + gpr_log(GPR_ERROR, "Completion queue next failed: %s", msg); - GRPC_ERROR_UNREF(err); - memset(&ret, 0, sizeof(ret)); - ret.type = GRPC_QUEUE_TIMEOUT; - dump_pending_tags(cc); - break; - } + GRPC_ERROR_UNREF(err); + memset(&ret, 0, sizeof(ret)); + ret.type = GRPC_QUEUE_TIMEOUT; + dump_pending_tags(cc); + break; } is_finished_arg.first_loop = false; del_plucker(cc, tag, &worker); diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d2a570cc87e..02328867aea 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -143,6 +143,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/time_averaged_stats.c', 'src/core/lib/iomgr/timer_generic.c', 'src/core/lib/iomgr/timer_heap.c', + 'src/core/lib/iomgr/timer_manager.c', 'src/core/lib/iomgr/timer_uv.c', 'src/core/lib/iomgr/udp_server.c', 'src/core/lib/iomgr/unix_sockets_posix.c', diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 29a79e7343d..751986d7ac2 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -238,39 +238,27 @@ class AsyncClient : public ClientImpl { void* got_tag; bool ok; - switch (cli_cqs_[thread_idx]->AsyncNext( - &got_tag, &ok, - std::chrono::system_clock::now() + std::chrono::milliseconds(10))) { - case CompletionQueue::GOT_EVENT: { - // Got a regular event, so process it - ClientRpcContext* ctx = ClientRpcContext::detag(got_tag); - // Proceed while holding a lock to make sure that - // this thread isn't supposed to shut down - std::lock_guard l(shutdown_state_[thread_idx]->mutex); - if (shutdown_state_[thread_idx]->shutdown) { - delete ctx; - return true; - } else if (!ctx->RunNextState(ok, entry)) { - // The RPC and callback are done, so clone the ctx - // and kickstart the new one - ctx->StartNewClone(cli_cqs_[thread_idx].get()); - // delete the old version - delete ctx; - } + if (cli_cqs_[thread_idx]->Next(&got_tag, &ok)) { + // Got a regular event, so process it + ClientRpcContext* ctx = ClientRpcContext::detag(got_tag); + // Proceed while holding a lock to make sure that + // this thread isn't supposed to shut down + std::lock_guard l(shutdown_state_[thread_idx]->mutex); + if (shutdown_state_[thread_idx]->shutdown) { + delete ctx; return true; + } else if (!ctx->RunNextState(ok, entry)) { + // The RPC and callback are done, so clone the ctx + // and kickstart the new one + ctx->StartNewClone(cli_cqs_[thread_idx].get()); + // delete the old version + delete ctx; } - case CompletionQueue::TIMEOUT: { - std::lock_guard l(shutdown_state_[thread_idx]->mutex); - if (shutdown_state_[thread_idx]->shutdown) { - return true; - } - return true; - } - case CompletionQueue::SHUTDOWN: // queue is shutting down, so we must be - // done - return true; + return true; + } else { + // queue is shutting down, so we must be done + return true; } - GPR_UNREACHABLE_CODE(return true); } std::vector> cli_cqs_; diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f4..e50d90648ab 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1026,6 +1026,8 @@ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_generic.h \ src/core/lib/iomgr/timer_heap.c \ src/core/lib/iomgr/timer_heap.h \ +src/core/lib/iomgr/timer_manager.c \ +src/core/lib/iomgr/timer_manager.h \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/timer_uv.h \ src/core/lib/iomgr/udp_server.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index f49c2de76c9..24737de8597 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1161,6 +1161,8 @@ src/core/lib/iomgr/timer_generic.c \ src/core/lib/iomgr/timer_generic.h \ src/core/lib/iomgr/timer_heap.c \ src/core/lib/iomgr/timer_heap.h \ +src/core/lib/iomgr/timer_manager.c \ +src/core/lib/iomgr/timer_manager.h \ src/core/lib/iomgr/timer_uv.c \ src/core/lib/iomgr/timer_uv.h \ src/core/lib/iomgr/udp_server.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index adaf5481b22..a9f6ac5cbd5 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7791,6 +7791,7 @@ "src/core/lib/iomgr/timer.h", "src/core/lib/iomgr/timer_generic.h", "src/core/lib/iomgr/timer_heap.h", + "src/core/lib/iomgr/timer_manager.h", "src/core/lib/iomgr/timer_uv.h", "src/core/lib/iomgr/udp_server.h", "src/core/lib/iomgr/unix_sockets_posix.h", @@ -7978,6 +7979,8 @@ "src/core/lib/iomgr/timer_generic.h", "src/core/lib/iomgr/timer_heap.c", "src/core/lib/iomgr/timer_heap.h", + "src/core/lib/iomgr/timer_manager.c", + "src/core/lib/iomgr/timer_manager.h", "src/core/lib/iomgr/timer_uv.c", "src/core/lib/iomgr/timer_uv.h", "src/core/lib/iomgr/udp_server.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a588..da9816f9a3e 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -437,6 +437,7 @@ + @@ -702,6 +703,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc2979..924576d35b9 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -322,6 +322,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -1031,6 +1034,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651c..3e4265bda29 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -431,6 +431,7 @@ + @@ -686,6 +687,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab36..c17d4deca4d 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -307,6 +307,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -998,6 +1001,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 71520098a6d..bb44a0772c8 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -361,6 +361,7 @@ + @@ -641,6 +642,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index de2dfe67e6a..35002b17560 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -202,6 +202,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -992,6 +995,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97c..b5605a93816 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -256,6 +256,7 @@ + @@ -475,6 +476,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4d..95e22922f49 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -259,6 +259,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -746,6 +749,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0bfda72e815..8b59aa82dd3 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -351,6 +351,7 @@ + @@ -608,6 +609,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 63c8d7f2542..2cec671ebe6 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -205,6 +205,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -902,6 +905,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From d4fc32eacd07a5b9ae9573d25dbee819266d761a Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 2 May 2017 14:15:12 -0700 Subject: [PATCH 109/177] New epoll-based polling engine with dedicated poller threads --- BUILD | 82 +- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + .../lib/iomgr/ev_epoll_thread_pool_linux.c | 1499 +++++++++++++++++ .../lib/iomgr/ev_epoll_thread_pool_linux.h | 42 + src/core/lib/iomgr/ev_posix.c | 2 + src/python/grpcio/grpc_core_dependencies.py | 1 + test/core/iomgr/pollset_set_test.c | 4 +- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 3 + tools/run_tests/run_tests.py | 2 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util.vcxproj.filters | 6 + .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + 28 files changed, 1672 insertions(+), 37 deletions(-) create mode 100644 src/core/lib/iomgr/ev_epoll_thread_pool_linux.c create mode 100644 src/core/lib/iomgr/ev_epoll_thread_pool_linux.h diff --git a/BUILD b/BUILD index 7b9e92e29fa..2a042fb1160 100644 --- a/BUILD +++ b/BUILD @@ -35,8 +35,12 @@ exports_files(["LICENSE"]) package(default_visibility = ["//visibility:public"]) -load("//bazel:grpc_build_system.bzl", "grpc_cc_library", - "grpc_proto_plugin", "grpc_cc_libraries") +load( + "//bazel:grpc_build_system.bzl", + "grpc_cc_library", + "grpc_proto_plugin", + "grpc_cc_libraries", +) # This should be updated along with build.yaml g_stands_for = "gregarious" @@ -55,10 +59,19 @@ grpc_cc_library( ) grpc_cc_libraries( - name_list = ["grpc", "grpc_unsecure",], srcs = [ "src/core/lib/surface/init.c", ], + additional_dep_list = [ + [ + "grpc_secure", + "grpc_resolver_dns_ares", + "grpc_lb_policy_grpclb_secure", + "grpc_transport_chttp2_client_secure", + "grpc_transport_chttp2_server_secure", + ], + [], + ], additional_src_list = [ [ "src/core/plugin_registry/grpc_plugin_registry.c", @@ -69,30 +82,24 @@ grpc_cc_libraries( ], ], language = "c", + name_list = [ + "grpc", + "grpc_unsecure", + ], standalone = True, deps = [ "census", "grpc_base", + "grpc_deadline_filter", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", "grpc_max_age_filter", + "grpc_message_size_filter", "grpc_resolver_dns_native", "grpc_resolver_sockaddr", "grpc_transport_chttp2_client_insecure", "grpc_transport_chttp2_server_insecure", - "grpc_message_size_filter", - "grpc_deadline_filter", - ], - additional_dep_list = [ - [ - "grpc_secure", - "grpc_resolver_dns_ares", - "grpc_lb_policy_grpclb_secure", - "grpc_transport_chttp2_client_secure", - "grpc_transport_chttp2_server_secure", - ], - [], ], ) @@ -105,9 +112,9 @@ grpc_cc_library( language = "c", deps = [ "grpc_base", + "grpc_http_filters", "grpc_transport_chttp2_client_secure", "grpc_transport_cronet_client_secure", - "grpc_http_filters", ], ) @@ -373,13 +380,13 @@ grpc_cc_library( hdrs = [ "src/core/lib/profiling/timers.h", "src/core/lib/support/arena.h", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", "src/core/lib/support/memory.h", - "src/core/lib/support/atomic.h", - "src/core/lib/support/atomic_with_atm.h", - "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.h", "src/core/lib/support/spinlock.h", @@ -466,6 +473,7 @@ grpc_cc_library( "src/core/lib/iomgr/endpoint_pair_windows.c", "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll_linux.c", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_posix.c", "src/core/lib/iomgr/exec_ctx.c", @@ -588,6 +596,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -791,16 +800,16 @@ grpc_cc_library( grpc_cc_library( name = "grpc_http_filters", - hdrs = [ - "src/core/ext/filters/http/message_compress/message_compress_filter.h", - "src/core/ext/filters/http/client/http_client_filter.h", - "src/core/ext/filters/http/server/http_server_filter.h", - ], srcs = [ - "src/core/ext/filters/http/message_compress/message_compress_filter.c", "src/core/ext/filters/http/client/http_client_filter.c", + "src/core/ext/filters/http/http_filters_plugin.c", + "src/core/ext/filters/http/message_compress/message_compress_filter.c", "src/core/ext/filters/http/server/http_server_filter.c", - "src/core/ext/filters/http/http_filters_plugin.c" + ], + hdrs = [ + "src/core/ext/filters/http/client/http_client_filter.h", + "src/core/ext/filters/http/message_compress/message_compress_filter.h", + "src/core/ext/filters/http/server/http_server_filter.h", ], language = "c", deps = [ @@ -1069,8 +1078,8 @@ grpc_cc_library( language = "c", deps = [ "grpc_base", - "grpc_transport_chttp2_alpn", "grpc_http_filters", + "grpc_transport_chttp2_alpn", ], ) @@ -1226,11 +1235,6 @@ grpc_cc_library( ) grpc_cc_libraries( - name_list = ["grpc++_base", "grpc++_base_unsecure"], - additional_dep_list = [ - ["grpc", ], - ["grpc_unsecure", ], - ], srcs = [ "src/cpp/client/channel_cc.cc", "src/cpp/client/client_context.cc", @@ -1265,7 +1269,7 @@ grpc_cc_libraries( "src/cpp/util/status.cc", "src/cpp/util/string_ref.cc", "src/cpp/util/time_cc.cc", - ], + ], hdrs = [ "src/cpp/client/create_channel_internal.h", "src/cpp/common/channel_filter.h", @@ -1274,8 +1278,16 @@ grpc_cc_libraries( "src/cpp/server/health/health.pb.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/thread_manager/thread_manager.h", - ], + ], + additional_dep_list = [ + ["grpc"], + ["grpc_unsecure"], + ], language = "c++", + name_list = [ + "grpc++_base", + "grpc++_base_unsecure", + ], public_hdrs = [ "include/grpc++/alarm.h", "include/grpc++/channel.h", @@ -1324,7 +1336,7 @@ grpc_cc_libraries( "include/grpc++/support/stub_options.h", "include/grpc++/support/sync_stream.h", "include/grpc++/support/time.h", - ], + ], deps = [ "grpc++_codegen_base", ], diff --git a/CMakeLists.txt b/CMakeLists.txt index 325344a81c9..c78dcce13a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -939,6 +939,7 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1265,6 +1266,7 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1576,6 +1578,7 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -1832,6 +1835,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2251,6 +2255,7 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -2576,6 +2581,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c @@ -3345,6 +3351,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll_linux.c + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c src/core/lib/iomgr/exec_ctx.c diff --git a/Makefile b/Makefile index 4597b6fe974..e93ac94db71 100644 --- a/Makefile +++ b/Makefile @@ -2922,6 +2922,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3246,6 +3247,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3556,6 +3558,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -3784,6 +3787,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4180,6 +4184,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -4513,6 +4518,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ @@ -5272,6 +5278,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/binding.gyp b/binding.gyp index 582c61282f9..2584c6af6b6 100644 --- a/binding.gyp +++ b/binding.gyp @@ -673,6 +673,7 @@ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/build.yaml b/build.yaml index 4cfa75cae14..887ca7646f7 100644 --- a/build.yaml +++ b/build.yaml @@ -198,6 +198,7 @@ filegroups: - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h - src/core/lib/iomgr/ev_epoll_linux.h + - src/core/lib/iomgr/ev_epoll_thread_pool_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h - src/core/lib/iomgr/exec_ctx.h @@ -305,6 +306,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll_linux.c + - src/core/lib/iomgr/ev_epoll_thread_pool_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c - src/core/lib/iomgr/exec_ctx.c diff --git a/config.m4 b/config.m4 index bbd667c9ec4..c478f77ba3f 100644 --- a/config.m4 +++ b/config.m4 @@ -109,6 +109,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll_linux.c \ + src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ src/core/lib/iomgr/exec_ctx.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 07755ac7279..b233df2febb 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -280,6 +280,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', @@ -486,6 +487,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', @@ -741,6 +743,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', 'src/core/lib/iomgr/ev_epoll_linux.h', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', 'src/core/lib/iomgr/exec_ctx.h', diff --git a/grpc.gemspec b/grpc.gemspec index 1cd6d66335e..783af00bb63 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -196,6 +196,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) + s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) s.files += %w( src/core/lib/iomgr/exec_ctx.h ) @@ -402,6 +403,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) + s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) s.files += %w( src/core/lib/iomgr/exec_ctx.c ) diff --git a/package.xml b/package.xml index e7d67eca180..00dc5aeb260 100644 --- a/package.xml +++ b/package.xml @@ -205,6 +205,7 @@ + @@ -411,6 +412,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c new file mode 100644 index 00000000000..7a9e20b1eaf --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -0,0 +1,1499 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" + +/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ +static int grpc_polling_trace = 0; /* Disabled by default */ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (grpc_polling_trace) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +struct polling_island; + +/******************************************************************************* + * Fd Declarations + */ +struct grpc_fd { + gpr_mu mu; + struct polling_island *pi; + + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Polling island Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define PI_UNREF(exec_ctx, p, r) \ + pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define PI_ADD_REF(p, r) pi_add_ref((p)) +#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ + +/* This is also used as grpc_workqueue (by directly casting it) */ +typedef struct polling_island { + grpc_closure_scheduler workqueue_scheduler; + + gpr_mu mu; + /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + the refcount. + Once the ref count becomes zero, this structure is destroyed which means + we should ensure that there is never a scenario where a PI_ADD_REF() is + racing with a PI_UNREF() that just made the ref_count zero. */ + gpr_atm ref_count; + + /* Number of threads currently polling on this island */ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* The fd of the underlying epoll set */ + int epoll_fd; + + /* The file descriptors in the epoll set */ + /* TODO: sreek - We no longer need this (and since no other structure in this + * polling engine keeps a reference to grpc_fd, we actually no longer need a + * ref count field in FD. Just a flag to say wheter it is orphaned or not */ + size_t fd_cnt; + size_t fd_capacity; + grpc_fd **fds; +} polling_island; + +/******************************************************************************* + * Pollset Declarations + */ +struct grpc_pollset_worker { + gpr_cv kick_cv; + + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; +}; + +struct grpc_pollset { + gpr_mu mu; + struct polling_island *pi; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ + gpr_atm is_shutdown; +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + void *no_op; +}; + +/***************************************************************************** + * Dedicated polling threads and pollsets - Declarations + */ + +size_t g_num_pollsets = 0; +struct grpc_pollset *g_pollsets = NULL; +gpr_thd_id *g_poller_threads = NULL; + +static void add_fd_to_global_pollset(grpc_fd *fd); +static void init_dedicated_pollsets(); +static void poller_thread_loop(void *arg); +static void start_dedicated_poller_threads(); +static void shutdown_dedicated_poller_threads(); + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Polling island Definitions + */ + +/* The wakeup fd that is used to wake up all threads in a Polling island. This + is useful in the polling island merge operation where we need to wakeup all + the threads currently polling the smaller polling island (so that they can + start polling the new/merged polling island) + + NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the + threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd polling_island_wakeup_fd; + +/* The polling island being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread polling_island *g_current_thread_polling_island; + +/* Forward declaration */ +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void pi_add_ref(polling_island *pi); +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void pi_add_ref_dbg(polling_island *pi, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_add_ref(pi); + gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, old_cnt + 1, reason, file, line); +} + +static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_unref(exec_ctx, pi); + gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_add_ref((polling_island *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_unref(exec_ctx, (polling_island *)workqueue); + } +} +#endif + +static void pi_add_ref(polling_island *pi) { + gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +} + +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { + /* If ref count went to zero, delete the polling island. This deletion is + not done under a lock since once the ref count goes to zero, we are + guaranteed that no one else holds a reference to the polling island (and + that there is no racing pi_add_ref() call either).*/ + if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { + polling_island_delete(exec_ctx, pi); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, + size_t fd_count, bool add_fd_refs, + grpc_error **error) { + int err; + size_t i; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "polling_island_add_fds"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + for (i = 0; i < fd_count; i++) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fds[i]; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); + if (err < 0) { + if (errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + continue; + } + + if (pi->fd_cnt == pi->fd_capacity) { + pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); + pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); + } + + pi->fds[pi->fd_cnt++] = fds[i]; + if (add_fd_refs) { + GRPC_FD_REF(fds[i], "polling_island"); + } + } +} + +/* The caller is expected to hold pi->mu before calling this */ +static void polling_island_add_wakeup_fd_locked(polling_island *pi, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "polling_island_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } + + for (i = 0; i < pi->fd_cnt; i++) { + if (pi->fds[i] == fd) { + pi->fds[i] = pi->fds[--pi->fd_cnt]; + GRPC_FD_UNREF(fd, "polling_island"); + break; + } + } +} + +/* Might return NULL in case of an error */ +static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, + grpc_fd *initial_fd, + grpc_error **error) { + polling_island *pi = NULL; + const char *err_desc = "polling_island_create"; + + *error = GRPC_ERROR_NONE; + + pi = gpr_malloc(sizeof(*pi)); + pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + gpr_mu_init(&pi->mu); + pi->fd_cnt = 0; + pi->fd_capacity = 0; + pi->fds = NULL; + pi->epoll_fd = -1; + + gpr_mu_init(&pi->workqueue_read_mu); + gpr_mpscq_init(&pi->workqueue_items); + gpr_atm_rel_store(&pi->workqueue_item_count, 0); + + gpr_atm_rel_store(&pi->ref_count, 0); + gpr_atm_rel_store(&pi->poller_count, 0); + + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (pi->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); + polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + + if (initial_fd != NULL) { + polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); + } + +done: + if (*error != GRPC_ERROR_NONE) { + polling_island_delete(exec_ctx, pi); + pi = NULL; + } + return pi; +} + +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { + GPR_ASSERT(pi->fd_cnt == 0); + + if (pi->epoll_fd >= 0) { + close(pi->epoll_fd); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); + gpr_mu_destroy(&pi->workqueue_read_mu); + gpr_mpscq_destroy(&pi->workqueue_items); + gpr_mu_destroy(&pi->mu); + grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + + gpr_free(pi->fds); + gpr_free(pi); +} + +static void workqueue_maybe_wakeup(polling_island *pi) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_polling_island == pi); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + } +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + polling_island *pi = (polling_island *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(pi); + } + + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + polling_island *pi = (polling_island *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &pi->workqueue_scheduler; +} + +static grpc_error *polling_island_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + } + + return error; +} + +static void polling_island_global_shutdown() { + grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->mu); + } + + /* Note: It is not really needed to get the new_fd->mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->mu); + new_fd->pi = NULL; + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + add_fd_to_global_pollset(new_fd); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->pi != NULL) { + polling_island *pi = fd->pi; + gpr_mu_lock(&pi->mu); + polling_island_remove_fd_locked(pi, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi->mu); + + unref_pi = fd->pi; + fd->pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + return NULL; +} + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); + +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->next = worker->prev = NULL; + gpr_cv_init(&worker->kick_cv); +} + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + gpr_cv_signal(&worker->kick_cv); + return GRPC_ERROR_NONE; +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->mu); + *mu = &pollset->mu; + pollset->pi = NULL; + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; + gpr_atm_no_barrier_store(&pollset->is_shutdown, 0); +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, + grpc_pollset *ps, char *reason) { + if (ps->pi != NULL) { + PI_UNREF(exec_ctx, ps->pi, reason); + } + ps->pi = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->pi to NULL */ + pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying polling island. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, + polling_island *pi) { + if (gpr_mu_trylock(&pi->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); + gpr_mu_unlock(&pi->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(pi); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(pi); + } + } + return false; +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_error **error) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + int timeout_ms = -1; + + GRPC_SCHEDULING_START_BLOCKING_REGION; + // gpr_log(GPR_ERROR, "epoll_wait(%d)..", epoll_fd); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); + /* gpr_log(GPR_ERROR, "epoll_wait(%d) returned: %d (errno: %d - %s)", + epoll_fd, ep_rv, errno, strerror(errno)); */ + + GRPC_SCHEDULING_END_BLOCKING_REGION; + + if (ep_rv < 0) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + gpr_atm_rel_store(&pollset->is_shutdown, 1); + gpr_log(GPR_INFO, "pollset poller: shutdown set"); + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + +static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, grpc_error **error) { + int epoll_fd = -1; + polling_island *pi = NULL; + GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + + /* Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. */ + + if (pollset->pi == NULL) { + pollset->pi = polling_island_create(exec_ctx, NULL, error); + if (pollset->pi == NULL) { + GPR_TIMER_END("pollset_work_and_unlock", 0); + return; /* Fatal error. Cannot continue */ + } + + PI_ADD_REF(pollset->pi, "ps"); + GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", + (void *)pollset, (void *)pollset->pi); + } + + pi = pollset->pi; + epoll_fd = pi->epoll_fd; + + /* Add an extra ref so that the island does not get destroyed (which means + the epoll_fd won't be closed) while we are are doing an epoll_wait() on the + epoll_fd */ + PI_ADD_REF(pi, "ps_work"); + gpr_mu_unlock(&pollset->mu); + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, pi)) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); + g_current_thread_polling_island = pi; + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, error); + g_current_thread_polling_island = NULL; + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + } + + /* Before leaving, release the extra ref we added to the polling island. It + is important to use "pi" here (i.e our old copy of pollset->pi + that we got before releasing the polling island lock). This is because + pollset->pi pointer might get udpated in other parts of the + code when there is an island merge while we are doing epoll_wait() above */ + PI_UNREF(exec_ctx, pi, "ps_work"); + + GPR_TIMER_END("pollset_work_and_unlock", 0); +} + +/* pollset->mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + + grpc_pollset_worker worker; + pollset_worker_init(&worker); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + push_front_worker(pollset, &worker); + + gpr_cv_wait(&worker.kick_cv, &pollset->mu, + gpr_convert_clock_type(deadline, GPR_CLOCK_REALTIME)); + /* pollset->mu locked here */ + + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, + grpc_fd *fd) { + GPR_TIMER_BEGIN("pollset_add_fd", 0); + + grpc_error *error = GRPC_ERROR_NONE; + polling_island *pi_new = NULL; + + gpr_mu_lock(&ps->mu); + gpr_mu_lock(&fd->mu); + + /* fd MUST have a NULL polling island */ + GPR_ASSERT(fd->pi == NULL); + + /* Early out if we are trying to add an 'fd' to a 'pollset' but the fd is + * already orphaned */ + if (fd->orphaned) { + gpr_mu_unlock(&ps->mu); + gpr_mu_unlock(&fd->mu); + return; + } + + pi_new = ps->pi; + if (pi_new == NULL) { + /* Unlock before creating a new polling island: the polling island will + create a workqueue which creates a file descriptor, and holding an fd + lock here can eventually cause a loop to appear to TSAN (making it + unhappy). We don't think it's a real loop (there's an epoch point + where that loop possibility disappears), but the advantages of + keeping TSAN happy outweigh any performance advantage we might have + by keeping the lock held. */ + gpr_mu_unlock(&fd->mu); + pi_new = polling_island_create(exec_ctx, fd, &error); + gpr_mu_lock(&fd->mu); + + GRPC_POLLING_TRACE( + "pollset_add_fd: Created new polling island: %p (ps: %p, fd: %d", + (void *)pi_new, (void *)ps, fd->fd); + } else { + gpr_mu_lock(&pi_new->mu); + polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); + gpr_mu_unlock(&pi_new->mu); + + GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", + (void *)pi_new, fd->fd); + } + + PI_ADD_REF(pi_new, "fd"); + fd->pi = pi_new; + + GPR_ASSERT((ps->pi == NULL) || (ps->pi == pi_new)); + if (ps->pi == NULL) { + PI_ADD_REF(pi_new, "pollset"); + ps->pi = pi_new; + } + + gpr_mu_unlock(&ps->mu); + gpr_mu_unlock(&fd->mu); + + GRPC_LOG_IF_ERROR("pollset_add_fd", error); + GPR_TIMER_END("pollset_add_fd", 0); +} + +static void pollset_add_fd_no_op(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + /* Nothing to do */ +} + +/******************************************************************************* + * Pollset-set Definitions + */ +grpc_pollset_set g_dummy_pollset_set; +static grpc_pollset_set *pollset_set_create(void) { + return &g_dummy_pollset_set; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + /* Nothing to do */ +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + shutdown_dedicated_poller_threads(); + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd_no_op, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/***************************************************************************** + * Dedicated polling threads and pollsets - Definitions + */ +static void add_fd_to_global_pollset(grpc_fd *fd) { + size_t idx = ((size_t)rand()) % g_num_pollsets; + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + pollset_add_fd(&exec_ctx, &g_pollsets[idx], fd); + grpc_exec_ctx_finish(&exec_ctx); +} + +static void init_dedicated_pollsets() { + gpr_mu *temp_mu; + + g_num_pollsets = (size_t)gpr_cpu_num_cores(); + g_pollsets = (grpc_pollset *)malloc(g_num_pollsets * sizeof(grpc_pollset)); + for (size_t i = 0; i < g_num_pollsets; i++) { + pollset_init(&g_pollsets[i], &temp_mu); + } + + gpr_log(GPR_INFO, "Created %ld pollsets", g_num_pollsets); +} + +static void poller_thread_loop(void *arg) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_error *error = GRPC_ERROR_NONE; + grpc_pollset *ps = (grpc_pollset *)arg; + + while (!gpr_atm_acq_load(&ps->is_shutdown)) { + gpr_mu_lock(&ps->mu); + pollset_work_and_unlock(&exec_ctx, ps, &error); + grpc_exec_ctx_flush(&exec_ctx); + } + + grpc_exec_ctx_finish(&exec_ctx); +} + +/* g_pollsets MUST be initialized before calling this */ +static void start_dedicated_poller_threads() { + GPR_ASSERT(g_pollsets); + gpr_log(GPR_ERROR, "Starting poller threads"); + + /* One thread per pollset */ + g_poller_threads = (gpr_thd_id *)malloc(g_num_pollsets * sizeof(gpr_thd_id)); + gpr_thd_options options = gpr_thd_options_default(); + gpr_thd_options_set_joinable(&options); + + for (size_t i = 0; i < g_num_pollsets; i++) { + gpr_thd_new(&g_poller_threads[i], poller_thread_loop, + (void *)&g_pollsets[i], &options); + } +} + +static void shutdown_dedicated_poller_threads() { + GPR_ASSERT(g_poller_threads); + grpc_error *error = GRPC_ERROR_NONE; + + gpr_log(GPR_INFO, "Shutting down pollers"); + + for (size_t i = 0; i < g_num_pollsets; i++) { + gpr_mu_lock(&g_pollsets[i].mu); + polling_island *pi = g_pollsets[i].pi; + GPR_ASSERT(pi); + gpr_mu_lock(&pi->mu); + polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); + gpr_mu_unlock(&pi->mu); + } + + for (size_t i = 0; i < g_num_pollsets; i++) { + gpr_thd_join(g_poller_threads[i]); + } +} + +/****************************************************************************/ + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + init_dedicated_pollsets(); + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + /* TODO (sreek): Maynot be a good idea to start threads here (especially if + * this engine doesn't get picked. Consider introducing an engine_init + * function in the vtable */ + start_dedicated_poller_threads(); + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h new file mode 100644 index 00000000000..6743dcfe3f6 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de83..4742b2e5719 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,6 +45,7 @@ #include #include "src/core/lib/iomgr/ev_epoll_linux.h" +#include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -66,6 +67,7 @@ typedef struct { static const event_engine_factory g_factories[] = { {"epoll", grpc_init_epoll_linux}, + {"epoll-threadpool", grpc_init_epoll_thread_pool_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, }; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d2a570cc87e..b249d2ee116 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -98,6 +98,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll_linux.c', + 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', 'src/core/lib/iomgr/exec_ctx.c', diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 3a9d459579f..89854407b19 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -449,7 +449,9 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_iomgr_init(); - if (poll_strategy != NULL && strcmp(poll_strategy, "epoll") == 0) { + if (poll_strategy != NULL && + (strcmp(poll_strategy, "epoll") == 0 || + strcmp(poll_strategy, "epoll-threadpool") == 0)) { pollset_set_test_basic(); pollset_set_test_dup_fds(); pollset_set_test_empty_pollset(); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f4..13543edb71d 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -941,6 +941,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index f49c2de76c9..de7a7e9c827 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1076,6 +1076,8 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ +src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index adaf5481b22..0bc4bd627e5 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7751,6 +7751,7 @@ "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -7893,6 +7894,8 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_linux.h", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", + "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.c", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4da2ba4c3b0..2cadb7f811c 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -72,7 +72,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { _POLLING_STRATEGIES = { - 'linux': ['epoll', 'poll', 'poll-cv'] + 'linux': ['epoll', 'epoll-threadpool', 'poll', 'poll-cv'] } diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a588..37d151b78a8 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -397,6 +397,7 @@ + @@ -612,6 +613,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc2979..16b3326d810 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -187,6 +187,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -911,6 +914,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651c..596893ed459 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -391,6 +391,7 @@ + @@ -596,6 +597,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab36..6e8e91fb554 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -172,6 +172,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -878,6 +881,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 71520098a6d..da34b36b6f6 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -321,6 +321,7 @@ + @@ -551,6 +552,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index de2dfe67e6a..d113dc9840c 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -67,6 +67,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -872,6 +875,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97c..2c8cfa7cadd 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -216,6 +216,7 @@ + @@ -385,6 +386,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4d..3e68ddc388b 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -124,6 +124,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -626,6 +629,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 0bfda72e815..d67c6c3d571 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -311,6 +311,7 @@ + @@ -518,6 +519,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 63c8d7f2542..9526515a44b 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -70,6 +70,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -782,6 +785,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From a03edfd2855779a672c396efe3d6c39edd390cf3 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 2 May 2017 16:26:24 -0700 Subject: [PATCH 110/177] Make sure dedicated pollsets have polling islands. Simplies a lot of code. Fix init/shutdown --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 135 ++++++++++-------- 1 file changed, 72 insertions(+), 63 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 7a9e20b1eaf..c91164a6296 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -221,7 +221,8 @@ struct grpc_pollset *g_pollsets = NULL; gpr_thd_id *g_poller_threads = NULL; static void add_fd_to_global_pollset(grpc_fd *fd); -static void init_dedicated_pollsets(); +static bool init_dedicated_pollsets(); +static void shutdown_dedicated_pollsets(); static void poller_thread_loop(void *arg); static void start_dedicated_poller_threads(); static void shutdown_dedicated_poller_threads(); @@ -258,7 +259,7 @@ static grpc_wakeup_fd polling_island_wakeup_fd; static __thread polling_island *g_current_thread_polling_island; /* Forward declaration */ -static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void polling_island_delete(polling_island *pi); static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error); @@ -337,7 +338,7 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { guaranteed that no one else holds a reference to the polling island (and that there is no racing pi_add_ref() call either).*/ if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { - polling_island_delete(exec_ctx, pi); + polling_island_delete(pi); } } @@ -442,8 +443,7 @@ static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, } /* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, - grpc_fd *initial_fd, +static polling_island *polling_island_create(grpc_fd *initial_fd, grpc_error **error) { polling_island *pi = NULL; const char *err_desc = "polling_island_create"; @@ -486,13 +486,13 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, done: if (*error != GRPC_ERROR_NONE) { - polling_island_delete(exec_ctx, pi); + polling_island_delete(pi); pi = NULL; } return pi; } -static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { +static void polling_island_delete(polling_island *pi) { GPR_ASSERT(pi->fd_cnt == 0); if (pi->epoll_fd >= 0) { @@ -807,9 +807,7 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); } -static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { - return NULL; -} +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; } /******************************************************************************* * Pollset Definitions @@ -1042,7 +1040,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; + const char *err_desc = "pollset_do_epoll_pwait"; int timeout_ms = -1; @@ -1095,27 +1093,29 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, } } +static void pollset_add_polling_island(grpc_pollset *ps, grpc_error **error) { + GPR_ASSERT(ps->pi == NULL); + ps->pi = polling_island_create(NULL, error); + if (ps->pi) { + PI_ADD_REF(ps->pi, "ps"); + GRPC_POLLING_TRACE( + "pollset_add_polling_island: pollset: %p created new pi: %p", + (void *)ps, (void *)ps->pi); + } +} + +/* Note: Make sure the pollset has a polling island (i.e pollset->pi != NULL) + * before calling this */ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_error **error) { + GPR_ASSERT(pollset->pi); + int epoll_fd = -1; polling_island *pi = NULL; GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the polling island. */ - - if (pollset->pi == NULL) { - pollset->pi = polling_island_create(exec_ctx, NULL, error); - if (pollset->pi == NULL) { - GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. Cannot continue */ - } - - PI_ADD_REF(pollset->pi, "ps"); - GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", - (void *)pollset, (void *)pollset->pi); - } - pi = pollset->pi; epoll_fd = pi->epoll_fd; @@ -1212,17 +1212,19 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, grpc_fd *fd) { + /* fd MUST have a NULL polling island and pollset MUST have a non-NULL polling + * island*/ + GPR_ASSERT(fd->pi == NULL); + GPR_ASSERT(ps->pi); + GPR_TIMER_BEGIN("pollset_add_fd", 0); grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi_new = NULL; + polling_island *pi = NULL; gpr_mu_lock(&ps->mu); gpr_mu_lock(&fd->mu); - /* fd MUST have a NULL polling island */ - GPR_ASSERT(fd->pi == NULL); - /* Early out if we are trying to add an 'fd' to a 'pollset' but the fd is * already orphaned */ if (fd->orphaned) { @@ -1231,39 +1233,16 @@ static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, return; } - pi_new = ps->pi; - if (pi_new == NULL) { - /* Unlock before creating a new polling island: the polling island will - create a workqueue which creates a file descriptor, and holding an fd - lock here can eventually cause a loop to appear to TSAN (making it - unhappy). We don't think it's a real loop (there's an epoch point - where that loop possibility disappears), but the advantages of - keeping TSAN happy outweigh any performance advantage we might have - by keeping the lock held. */ - gpr_mu_unlock(&fd->mu); - pi_new = polling_island_create(exec_ctx, fd, &error); - gpr_mu_lock(&fd->mu); + pi = ps->pi; + gpr_mu_lock(&pi->mu); + polling_island_add_fds_locked(pi, &fd, 1, true, &error); + gpr_mu_unlock(&pi->mu); - GRPC_POLLING_TRACE( - "pollset_add_fd: Created new polling island: %p (ps: %p, fd: %d", - (void *)pi_new, (void *)ps, fd->fd); - } else { - gpr_mu_lock(&pi_new->mu); - polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); - gpr_mu_unlock(&pi_new->mu); + PI_ADD_REF(pi, "fd"); + fd->pi = pi; - GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", - (void *)pi_new, fd->fd); - } - - PI_ADD_REF(pi_new, "fd"); - fd->pi = pi_new; - - GPR_ASSERT((ps->pi == NULL) || (ps->pi == pi_new)); - if (ps->pi == NULL) { - PI_ADD_REF(pi_new, "pollset"); - ps->pi = pi_new; - } + GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", (void *)pi, + fd->fd); gpr_mu_unlock(&ps->mu); gpr_mu_unlock(&fd->mu); @@ -1328,6 +1307,7 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { shutdown_dedicated_poller_threads(); + shutdown_dedicated_pollsets(); fd_global_shutdown(); pollset_global_shutdown(); polling_island_global_shutdown(); @@ -1381,16 +1361,38 @@ static void add_fd_to_global_pollset(grpc_fd *fd) { grpc_exec_ctx_finish(&exec_ctx); } -static void init_dedicated_pollsets() { +static bool init_dedicated_pollsets() { gpr_mu *temp_mu; + grpc_error *error = GRPC_ERROR_NONE; + bool is_success = true; g_num_pollsets = (size_t)gpr_cpu_num_cores(); g_pollsets = (grpc_pollset *)malloc(g_num_pollsets * sizeof(grpc_pollset)); + for (size_t i = 0; i < g_num_pollsets; i++) { pollset_init(&g_pollsets[i], &temp_mu); + pollset_add_polling_island(&g_pollsets[i], &error); + if (g_pollsets[i].pi == NULL) { + is_success = false; + break; + } + } + + if (is_success) { + gpr_log(GPR_INFO, "Created %ld dedicated pollsets", g_num_pollsets); + } else { + shutdown_dedicated_pollsets(); } - gpr_log(GPR_INFO, "Created %ld pollsets", g_num_pollsets); + GRPC_LOG_IF_ERROR("init_dedicated_pollsets", error); + return is_success; +} + +static void shutdown_dedicated_pollsets() { + if (g_pollsets) { + gpr_free(g_pollsets); + g_pollsets = NULL; + } } static void poller_thread_loop(void *arg) { @@ -1405,6 +1407,7 @@ static void poller_thread_loop(void *arg) { } grpc_exec_ctx_finish(&exec_ctx); + GRPC_LOG_IF_ERROR("poller_thread_loop", error); } /* g_pollsets MUST be initialized before calling this */ @@ -1425,6 +1428,7 @@ static void start_dedicated_poller_threads() { static void shutdown_dedicated_poller_threads() { GPR_ASSERT(g_poller_threads); + GPR_ASSERT(g_pollsets); grpc_error *error = GRPC_ERROR_NONE; gpr_log(GPR_INFO, "Shutting down pollers"); @@ -1432,7 +1436,6 @@ static void shutdown_dedicated_poller_threads() { for (size_t i = 0; i < g_num_pollsets; i++) { gpr_mu_lock(&g_pollsets[i].mu); polling_island *pi = g_pollsets[i].pi; - GPR_ASSERT(pi); gpr_mu_lock(&pi->mu); polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); gpr_mu_unlock(&pi->mu); @@ -1441,6 +1444,10 @@ static void shutdown_dedicated_poller_threads() { for (size_t i = 0; i < g_num_pollsets; i++) { gpr_thd_join(g_poller_threads[i]); } + + GRPC_LOG_IF_ERROR("shutdown_dedicated_poller_threads", error); + gpr_free(g_poller_threads); + g_poller_threads = NULL; } /****************************************************************************/ @@ -1469,7 +1476,9 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - init_dedicated_pollsets(); + if (!init_dedicated_pollsets()) { + return NULL; + } fd_global_init(); From 50f85f726b4f011f491e90f428f2d116947224e3 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 2 May 2017 19:44:28 -0700 Subject: [PATCH 111/177] More simplifications --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 402 ++++++------------ .../lib/iomgr/ev_epoll_thread_pool_linux.h | 2 +- 2 files changed, 140 insertions(+), 264 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index c91164a6296..055b31331d5 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -104,28 +104,9 @@ struct grpc_fd { struct grpc_fd *freelist_next; grpc_closure *on_done_closure; - /* The pollset that last noticed that the fd is readable. The actual type - * stored in this is (grpc_pollset *) */ - gpr_atm read_notifier_pollset; - grpc_iomgr_object iomgr_object; }; -/* Reference counting for fds */ -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line); -#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) -#else -static void fd_ref(grpc_fd *fd); -static void fd_unref(grpc_fd *fd); -#define GRPC_FD_REF(fd, reason) fd_ref(fd) -#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) -#endif - static void fd_global_init(void); static void fd_global_shutdown(void); @@ -150,12 +131,11 @@ static void fd_global_shutdown(void); typedef struct polling_island { grpc_closure_scheduler workqueue_scheduler; - gpr_mu mu; /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement - the refcount. - Once the ref count becomes zero, this structure is destroyed which means - we should ensure that there is never a scenario where a PI_ADD_REF() is - racing with a PI_UNREF() that just made the ref_count zero. */ + the refcount. Once the ref count becomes zero, this structure is destroyed + which means we should ensure that there is never a scenario where a + PI_ADD_REF() is racing with a PI_UNREF() that just made the ref_count + zero. */ gpr_atm ref_count; /* Number of threads currently polling on this island */ @@ -170,16 +150,11 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; + /* Is the polling island shutdown */ + gpr_atm is_shutdown; + /* The fd of the underlying epoll set */ int epoll_fd; - - /* The file descriptors in the epoll set */ - /* TODO: sreek - We no longer need this (and since no other structure in this - * polling engine keeps a reference to grpc_fd, we actually no longer need a - * ref count field in FD. Just a flag to say wheter it is orphaned or not */ - size_t fd_cnt; - size_t fd_capacity; - grpc_fd **fds; } polling_island; /******************************************************************************* @@ -202,7 +177,6 @@ struct grpc_pollset { bool shutting_down; /* Is the pollset shutting down ? */ bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ grpc_closure *shutdown_done; /* Called after after shutdown is complete */ - gpr_atm is_shutdown; }; /******************************************************************************* @@ -216,13 +190,19 @@ struct grpc_pollset_set { * Dedicated polling threads and pollsets - Declarations */ -size_t g_num_pollsets = 0; -struct grpc_pollset *g_pollsets = NULL; +size_t g_num_pi = 1; +struct polling_island **g_polling_islands = NULL; +size_t g_num_threads_per_pi = 1; gpr_thd_id *g_poller_threads = NULL; -static void add_fd_to_global_pollset(grpc_fd *fd); -static bool init_dedicated_pollsets(); -static void shutdown_dedicated_pollsets(); +/* Used as read-notifier pollsets for fds. We won't be using read notifier + * pollsets with this polling engine. So it does not matter what pollset we + * return */ +grpc_pollset g_read_notifier; + +static void add_fd_to_dedicated_pi(grpc_fd *fd); +static bool init_dedicated_polling_islands(); +static void shutdown_dedicated_polling_islands(); static void poller_thread_loop(void *arg); static void start_dedicated_poller_threads(); static void shutdown_dedicated_poller_threads(); @@ -342,51 +322,31 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { } } -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, - size_t fd_count, bool add_fd_refs, - grpc_error **error) { +static void polling_island_add_fd_locked(polling_island *pi, grpc_fd *fd, + grpc_error **error) { int err; - size_t i; struct epoll_event ev; char *err_msg; - const char *err_desc = "polling_island_add_fds"; + const char *err_desc = "polling_island_add_fd_locked"; #ifdef GRPC_TSAN /* See the definition of g_epoll_sync for more context */ gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); #endif /* defined(GRPC_TSAN) */ - for (i = 0; i < fd_count; i++) { - ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); - ev.data.ptr = fds[i]; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); - if (err < 0) { - if (errno != EEXIST) { - gpr_asprintf( - &err_msg, - "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", - pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - gpr_free(err_msg); - } - - continue; - } - - if (pi->fd_cnt == pi->fd_capacity) { - pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); - pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); - } - - pi->fds[pi->fd_cnt++] = fds[i]; - if (add_fd_refs) { - GRPC_FD_REF(fds[i], "polling_island"); - } + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); } } -/* The caller is expected to hold pi->mu before calling this */ static void polling_island_add_wakeup_fd_locked(polling_island *pi, grpc_wakeup_fd *wakeup_fd, grpc_error **error) { @@ -410,12 +370,9 @@ static void polling_island_add_wakeup_fd_locked(polling_island *pi, } } -/* The caller is expected to hold pi->mu lock before calling this function */ -static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, - bool is_fd_closed, - grpc_error **error) { +static void polling_island_remove_fd(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, grpc_error **error) { int err; - size_t i; char *err_msg; const char *err_desc = "polling_island_remove_fd"; @@ -432,19 +389,10 @@ static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, gpr_free(err_msg); } } - - for (i = 0; i < pi->fd_cnt; i++) { - if (pi->fds[i] == fd) { - pi->fds[i] = pi->fds[--pi->fd_cnt]; - GRPC_FD_UNREF(fd, "polling_island"); - break; - } - } } /* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_fd *initial_fd, - grpc_error **error) { +static polling_island *polling_island_create(grpc_error **error) { polling_island *pi = NULL; const char *err_desc = "polling_island_create"; @@ -452,10 +400,6 @@ static polling_island *polling_island_create(grpc_fd *initial_fd, pi = gpr_malloc(sizeof(*pi)); pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - gpr_mu_init(&pi->mu); - pi->fd_cnt = 0; - pi->fd_capacity = 0; - pi->fds = NULL; pi->epoll_fd = -1; gpr_mu_init(&pi->workqueue_read_mu); @@ -465,6 +409,8 @@ static polling_island *polling_island_create(grpc_fd *initial_fd, gpr_atm_rel_store(&pi->ref_count, 0); gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&pi->is_shutdown, false); + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), err_desc)) { goto done; @@ -480,10 +426,6 @@ static polling_island *polling_island_create(grpc_fd *initial_fd, polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); - if (initial_fd != NULL) { - polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); - } - done: if (*error != GRPC_ERROR_NONE) { polling_island_delete(pi); @@ -493,18 +435,15 @@ done: } static void polling_island_delete(polling_island *pi) { - GPR_ASSERT(pi->fd_cnt == 0); - if (pi->epoll_fd >= 0) { close(pi->epoll_fd); } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); gpr_mu_destroy(&pi->workqueue_read_mu); gpr_mpscq_destroy(&pi->workqueue_items); - gpr_mu_destroy(&pi->mu); grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); - gpr_free(pi->fds); gpr_free(pi); } @@ -590,6 +529,7 @@ static void polling_island_global_shutdown() { static grpc_fd *fd_freelist = NULL; static gpr_mu fd_freelist_mu; +// #define GRPC_FD_REF_COUNT_DEBUG #ifdef GRPC_FD_REF_COUNT_DEBUG #define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) #define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) @@ -634,22 +574,6 @@ static void unref_by(grpc_fd *fd, int n) { } } -/* Increment refcount by two to avoid changing the orphan bit */ -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void fd_ref(grpc_fd *fd, const char *reason, const char *file, - int line) { - ref_by(fd, 2, reason, file, line); -} - -static void fd_unref(grpc_fd *fd, const char *reason, const char *file, - int line) { - unref_by(fd, 2, reason, file, line); -} -#else -static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } -static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } -#endif - static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } static void fd_global_shutdown(void) { @@ -690,7 +614,6 @@ static grpc_fd *fd_create(int fd, const char *name) { new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); grpc_lfev_init(&new_fd->write_closure); - gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = NULL; new_fd->on_done_closure = NULL; @@ -704,7 +627,9 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); #endif gpr_free(fd_name); - add_fd_to_global_pollset(new_fd); + + /* Associate the fd with one of the dedicated pi */ + add_fd_to_dedicated_pi(new_fd); return new_fd; } @@ -744,20 +669,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, to be alive (and not added to freelist) until the end of this function */ REF_BY(fd, 1, reason); - /* Remove the fd from the polling island: - - Get a lock on the latest polling island (i.e the last island in the - linked list pointed by fd->pi). This is the island that - would actually contain the fd - - Remove the fd from the latest polling island - - Unlock the latest polling island - - Set fd->pi to NULL (but remove the ref on the polling island - before doing this.) */ + /* Remove the fd from the polling island */ if (fd->pi != NULL) { polling_island *pi = fd->pi; - gpr_mu_lock(&pi->mu); - polling_island_remove_fd_locked(pi, fd, is_fd_closed, &error); - gpr_mu_unlock(&pi->mu); - + polling_island_remove_fd(pi, fd, is_fd_closed, &error); unref_pi = fd->pi; fd->pi = NULL; } @@ -777,10 +692,11 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, GRPC_ERROR_UNREF(error); } +/* This polling engine doesn't really need the read notifier functionality. So + * it just returns a dummy read notifier pollset */ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { - gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); - return (grpc_pollset *)notifier; + return &g_read_notifier; } static bool fd_is_shutdown(grpc_fd *fd) { @@ -812,6 +728,7 @@ static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { return NULL; } /******************************************************************************* * Pollset Definitions */ +/* TODO: sreek - Not needed anymore */ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); @@ -938,20 +855,10 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->shutting_down = false; pollset->finish_shutdown_called = false; pollset->shutdown_done = NULL; - gpr_atm_no_barrier_store(&pollset->is_shutdown, 0); } -static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, - grpc_pollset *notifier) { +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->read_closure); - - /* Note, it is possible that fd_become_readable might be called twice with - different 'notifier's when an fd becomes readable and it is in two epoll - sets (This can happen briefly during polling island merges). In such cases - it does not really matter which notifer is set as the read_notifier_pollset - (They would both point to the same polling island anyway) */ - /* Use release store to match with acquire load in fd_get_read_notifier */ - gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); } static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { @@ -1034,22 +941,17 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, } #define GRPC_EPOLL_MAX_EVENTS 100 -static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, - grpc_pollset *pollset, polling_island *pi, - grpc_error **error) { +static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, + polling_island *pi, grpc_error **error) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; - const char *err_desc = "pollset_do_epoll_pwait"; + const char *err_desc = "do_epoll_wait"; int timeout_ms = -1; GRPC_SCHEDULING_START_BLOCKING_REGION; - // gpr_log(GPR_ERROR, "epoll_wait(%d)..", epoll_fd); ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); - /* gpr_log(GPR_ERROR, "epoll_wait(%d) returned: %d (errno: %d - %s)", - epoll_fd, ep_rv, errno, strerror(errno)); */ - GRPC_SCHEDULING_END_BLOCKING_REGION; if (ep_rv < 0) { @@ -1076,7 +978,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, err_desc); maybe_do_workqueue_work(exec_ctx, pi); } else if (data_ptr == &polling_island_wakeup_fd) { - gpr_atm_rel_store(&pollset->is_shutdown, 1); + gpr_atm_rel_store(&pi->is_shutdown, 1); gpr_log(GPR_INFO, "pollset poller: shutdown set"); } else { grpc_fd *fd = data_ptr; @@ -1084,7 +986,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); int write_ev = ep_ev[i].events & EPOLLOUT; if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); + fd_become_readable(exec_ctx, fd); } if (write_ev || cancel) { fd_become_writable(exec_ctx, fd); @@ -1093,37 +995,19 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, } } -static void pollset_add_polling_island(grpc_pollset *ps, grpc_error **error) { - GPR_ASSERT(ps->pi == NULL); - ps->pi = polling_island_create(NULL, error); - if (ps->pi) { - PI_ADD_REF(ps->pi, "ps"); - GRPC_POLLING_TRACE( - "pollset_add_polling_island: pollset: %p created new pi: %p", - (void *)ps, (void *)ps->pi); - } -} - -/* Note: Make sure the pollset has a polling island (i.e pollset->pi != NULL) - * before calling this */ -static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, - grpc_pollset *pollset, grpc_error **error) { - GPR_ASSERT(pollset->pi); - +static void polling_island_work(grpc_exec_ctx *exec_ctx, polling_island *pi, + grpc_error **error) { int epoll_fd = -1; - polling_island *pi = NULL; - GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + GPR_TIMER_BEGIN("polling_island_work", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the polling island. */ - pi = pollset->pi; epoll_fd = pi->epoll_fd; /* Add an extra ref so that the island does not get destroyed (which means the epoll_fd won't be closed) while we are are doing an epoll_wait() on the epoll_fd */ PI_ADD_REF(pi, "ps_work"); - gpr_mu_unlock(&pollset->mu); /* If we get some workqueue work to do, it might end up completing an item on the completion queue, so there's no need to poll... so we skip that and @@ -1131,7 +1015,9 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, if (!maybe_do_workqueue_work(exec_ctx, pi)) { gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); g_current_thread_polling_island = pi; - pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, error); + + do_epoll_wait(exec_ctx, epoll_fd, pi, error); + g_current_thread_polling_island = NULL; gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); } @@ -1143,7 +1029,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, code when there is an island merge while we are doing epoll_wait() above */ PI_UNREF(exec_ctx, pi, "ps_work"); - GPR_TIMER_END("pollset_work_and_unlock", 0); + GPR_TIMER_END("polling_island_work", 0); } /* pollset->mu lock must be held by the caller before calling this. @@ -1210,49 +1096,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, return error; } -static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) { - /* fd MUST have a NULL polling island and pollset MUST have a non-NULL polling - * island*/ - GPR_ASSERT(fd->pi == NULL); - GPR_ASSERT(ps->pi); - - GPR_TIMER_BEGIN("pollset_add_fd", 0); - - grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi = NULL; - - gpr_mu_lock(&ps->mu); - gpr_mu_lock(&fd->mu); - - /* Early out if we are trying to add an 'fd' to a 'pollset' but the fd is - * already orphaned */ - if (fd->orphaned) { - gpr_mu_unlock(&ps->mu); - gpr_mu_unlock(&fd->mu); - return; - } - - pi = ps->pi; - gpr_mu_lock(&pi->mu); - polling_island_add_fds_locked(pi, &fd, 1, true, &error); - gpr_mu_unlock(&pi->mu); - - PI_ADD_REF(pi, "fd"); - fd->pi = pi; - - GRPC_POLLING_TRACE("pollset_add_fd: ps->pi = %p. Add fd: %d", (void *)pi, - fd->fd); - - gpr_mu_unlock(&ps->mu); - gpr_mu_unlock(&fd->mu); - - GRPC_LOG_IF_ERROR("pollset_add_fd", error); - GPR_TIMER_END("pollset_add_fd", 0); -} - -static void pollset_add_fd_no_op(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_fd *fd) { /* Nothing to do */ } @@ -1307,10 +1152,11 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { shutdown_dedicated_poller_threads(); - shutdown_dedicated_pollsets(); + shutdown_dedicated_polling_islands(); fd_global_shutdown(); pollset_global_shutdown(); polling_island_global_shutdown(); + gpr_log(GPR_INFO, "ev-epoll-threadpool engine shutdown complete"); } static const grpc_event_engine_vtable vtable = { @@ -1331,7 +1177,7 @@ static const grpc_event_engine_vtable vtable = { .pollset_destroy = pollset_destroy, .pollset_work = pollset_work, .pollset_kick = pollset_kick, - .pollset_add_fd = pollset_add_fd_no_op, + .pollset_add_fd = pollset_add_fd, .pollset_set_create = pollset_set_create, .pollset_set_destroy = pollset_set_destroy, @@ -1354,55 +1200,85 @@ static const grpc_event_engine_vtable vtable = { /***************************************************************************** * Dedicated polling threads and pollsets - Definitions */ -static void add_fd_to_global_pollset(grpc_fd *fd) { - size_t idx = ((size_t)rand()) % g_num_pollsets; - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - pollset_add_fd(&exec_ctx, &g_pollsets[idx], fd); - grpc_exec_ctx_finish(&exec_ctx); +static void add_fd_to_dedicated_pi(grpc_fd *fd) { + GPR_ASSERT(fd->pi == NULL); + GPR_TIMER_BEGIN("add_fd_to_dedicated_pi", 0); + + grpc_error *error = GRPC_ERROR_NONE; + size_t idx = ((size_t)rand()) % g_num_pi; + polling_island *pi = g_polling_islands[idx]; + + gpr_mu_lock(&fd->mu); + + if (fd->orphaned) { + gpr_mu_unlock(&fd->mu); + return; /* Early out */ + } + + polling_island_add_fd_locked(pi, fd, &error); + PI_ADD_REF(pi, "fd"); + fd->pi = pi; + + GRPC_POLLING_TRACE("add_fd_to_dedicated_pi (fd: %d, pi idx = %ld)", fd->fd, + idx); + gpr_mu_unlock(&fd->mu); + + GRPC_LOG_IF_ERROR("add_fd_to_dedicated_pi", error); + GPR_TIMER_END("add_fd_to_dedicated_pi", 0); } -static bool init_dedicated_pollsets() { - gpr_mu *temp_mu; +static bool init_dedicated_polling_islands() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; - g_num_pollsets = (size_t)gpr_cpu_num_cores(); - g_pollsets = (grpc_pollset *)malloc(g_num_pollsets * sizeof(grpc_pollset)); + g_polling_islands = + (polling_island **)malloc(g_num_pi * sizeof(polling_island *)); - for (size_t i = 0; i < g_num_pollsets; i++) { - pollset_init(&g_pollsets[i], &temp_mu); - pollset_add_polling_island(&g_pollsets[i], &error); - if (g_pollsets[i].pi == NULL) { + for (size_t i = 0; i < g_num_pi; i++) { + g_polling_islands[i] = polling_island_create(&error); + if (g_polling_islands[i] == NULL) { + gpr_log(GPR_ERROR, "Error in creating a dedicated polling island"); + g_num_pi = i; /* Helps cleanup */ + shutdown_dedicated_polling_islands(); is_success = false; - break; + goto done; } - } - if (is_success) { - gpr_log(GPR_INFO, "Created %ld dedicated pollsets", g_num_pollsets); - } else { - shutdown_dedicated_pollsets(); + PI_ADD_REF(g_polling_islands[i], "init_dedicated_polling_islands"); } - GRPC_LOG_IF_ERROR("init_dedicated_pollsets", error); + gpr_mu *mu; + pollset_init(&g_read_notifier, &mu); + +done: + GRPC_LOG_IF_ERROR("init_dedicated_polling_islands", error); return is_success; } -static void shutdown_dedicated_pollsets() { - if (g_pollsets) { - gpr_free(g_pollsets); - g_pollsets = NULL; +static void shutdown_dedicated_polling_islands() { + if (!g_polling_islands) { + return; } + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + for (size_t i = 0; i < g_num_pi; i++) { + PI_UNREF(&exec_ctx, g_polling_islands[i], + "shutdown_dedicated_polling_islands"); + } + grpc_exec_ctx_finish(&exec_ctx); + + gpr_free(g_polling_islands); + g_polling_islands = NULL; + pollset_destroy(&g_read_notifier); } static void poller_thread_loop(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_error *error = GRPC_ERROR_NONE; - grpc_pollset *ps = (grpc_pollset *)arg; + polling_island *pi = (polling_island *)arg; - while (!gpr_atm_acq_load(&ps->is_shutdown)) { - gpr_mu_lock(&ps->mu); - pollset_work_and_unlock(&exec_ctx, ps, &error); + while (!gpr_atm_acq_load(&pi->is_shutdown)) { + polling_island_work(&exec_ctx, pi, &error); grpc_exec_ctx_flush(&exec_ctx); } @@ -1410,41 +1286,41 @@ static void poller_thread_loop(void *arg) { GRPC_LOG_IF_ERROR("poller_thread_loop", error); } -/* g_pollsets MUST be initialized before calling this */ +/* g_polling_islands MUST be initialized before calling this */ static void start_dedicated_poller_threads() { - GPR_ASSERT(g_pollsets); - gpr_log(GPR_ERROR, "Starting poller threads"); + GPR_ASSERT(g_polling_islands); + + gpr_log(GPR_INFO, "Starting poller threads"); /* One thread per pollset */ - g_poller_threads = (gpr_thd_id *)malloc(g_num_pollsets * sizeof(gpr_thd_id)); + g_poller_threads = (gpr_thd_id *)malloc(g_num_pi * sizeof(gpr_thd_id)); gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); - for (size_t i = 0; i < g_num_pollsets; i++) { + for (size_t i = 0; i < g_num_pi; i++) { gpr_thd_new(&g_poller_threads[i], poller_thread_loop, - (void *)&g_pollsets[i], &options); + (void *)g_polling_islands[i], &options); } } static void shutdown_dedicated_poller_threads() { GPR_ASSERT(g_poller_threads); - GPR_ASSERT(g_pollsets); + GPR_ASSERT(g_polling_islands); grpc_error *error = GRPC_ERROR_NONE; gpr_log(GPR_INFO, "Shutting down pollers"); - for (size_t i = 0; i < g_num_pollsets; i++) { - gpr_mu_lock(&g_pollsets[i].mu); - polling_island *pi = g_pollsets[i].pi; - gpr_mu_lock(&pi->mu); + polling_island *pi = NULL; + for (size_t i = 0; i < g_num_pi; i++) { + pi = g_polling_islands[i]; polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); - gpr_mu_unlock(&pi->mu); } - for (size_t i = 0; i < g_num_pollsets; i++) { + for (size_t i = 0; i < g_num_pi; i++) { gpr_thd_join(g_poller_threads[i]); } + gpr_log(GPR_ERROR, "polling island delete called"); GRPC_LOG_IF_ERROR("shutdown_dedicated_poller_threads", error); gpr_free(g_poller_threads); g_poller_threads = NULL; @@ -1476,10 +1352,6 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!init_dedicated_pollsets()) { - return NULL; - } - fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { @@ -1491,6 +1363,10 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } + if (!init_dedicated_polling_islands()) { + return NULL; + } + /* TODO (sreek): Maynot be a good idea to start threads here (especially if * this engine doesn't get picked. Consider introducing an engine_init * function in the vtable */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h index 6743dcfe3f6..f4959e3fee7 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2017, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 8ed56f5a4b63157c98d579bca114ac355357fc82 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 09:59:25 -0700 Subject: [PATCH 112/177] Remove refcnt from fd --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 112 ++++++------------ 1 file changed, 35 insertions(+), 77 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 055b31331d5..96e8000da4e 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -72,6 +72,11 @@ static int grpc_polling_trace = 0; /* Disabled by default */ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + /* TODO: sreek: Right now, this wakes up all pollers. In future we should make * sure to wake up one polling thread (which can wake up other threads if * needed) */ @@ -87,15 +92,9 @@ struct grpc_fd { struct polling_island *pi; int fd; - /* refst format: - bit 0 : 1=Active / 0=Orphaned - bits 1-n : refcount - Ref/Unref by two to avoid altering the orphaned bit */ - gpr_atm refst; - - /* The fd is either closed or we relinquished control of it. In either - cases, this indicates that the 'fd' on this structure is no longer - valid */ + + /* The fd is either closed or we relinquished control of it. In either cases, + this indicates that the 'fd' on this structure is no longer valid */ bool orphaned; gpr_atm read_closure; @@ -182,9 +181,7 @@ struct grpc_pollset { /******************************************************************************* * Pollset-set Declarations */ -struct grpc_pollset_set { - void *no_op; -}; +struct grpc_pollset_set {}; /***************************************************************************** * Dedicated polling threads and pollsets - Declarations @@ -521,57 +518,31 @@ static void polling_island_global_shutdown() { * becomes a spurious read notification on a reused fd. */ -/* The alarm system needs to be able to wakeup 'some poller' sometimes - * (specifically when a new alarm needs to be triggered earlier than the next - * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a - * case occurs. */ - static grpc_fd *fd_freelist = NULL; static gpr_mu fd_freelist_mu; -// #define GRPC_FD_REF_COUNT_DEBUG -#ifdef GRPC_FD_REF_COUNT_DEBUG -#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) -#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) -static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); -#else -#define REF_BY(fd, n, reason) ref_by(fd, n) -#define UNREF_BY(fd, n, reason) unref_by(fd, n) -static void ref_by(grpc_fd *fd, int n) { -#endif - GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); -} +static grpc_fd *get_fd_from_freelist() { + grpc_fd *new_fd = NULL; -#ifdef GRPC_FD_REF_COUNT_DEBUG -static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, - int line) { - gpr_atm old; - gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, - (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), - gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); -#else -static void unref_by(grpc_fd *fd, int n) { - gpr_atm old; -#endif - old = gpr_atm_full_fetch_add(&fd->refst, -n); - if (old == n) { - /* Add the fd to the freelist */ - gpr_mu_lock(&fd_freelist_mu); - fd->freelist_next = fd_freelist; - fd_freelist = fd; - grpc_iomgr_unregister_object(&fd->iomgr_object); - - grpc_lfev_destroy(&fd->read_closure); - grpc_lfev_destroy(&fd->write_closure); - - gpr_mu_unlock(&fd_freelist_mu); - } else { - GPR_ASSERT(old > n); + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; } + gpr_mu_unlock(&fd_freelist_mu); + return new_fd; +} + +static void add_fd_to_freelist(grpc_fd *fd) { + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); } static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } @@ -589,15 +560,7 @@ static void fd_global_shutdown(void) { } static grpc_fd *fd_create(int fd, const char *name) { - grpc_fd *new_fd = NULL; - - gpr_mu_lock(&fd_freelist_mu); - if (fd_freelist != NULL) { - new_fd = fd_freelist; - fd_freelist = fd_freelist->freelist_next; - } - gpr_mu_unlock(&fd_freelist_mu); - + grpc_fd *new_fd = get_fd_from_freelist(); if (new_fd == NULL) { new_fd = gpr_malloc(sizeof(grpc_fd)); gpr_mu_init(&new_fd->mu); @@ -609,7 +572,6 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_mu_lock(&new_fd->mu); new_fd->pi = NULL; - gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); new_fd->fd = fd; new_fd->orphaned = false; grpc_lfev_init(&new_fd->read_closure); @@ -623,9 +585,7 @@ static grpc_fd *fd_create(int fd, const char *name) { char *fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); -#ifdef GRPC_FD_REF_COUNT_DEBUG gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); -#endif gpr_free(fd_name); /* Associate the fd with one of the dedicated pi */ @@ -665,14 +625,9 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; - /* Remove the active status but keep referenced. We want this grpc_fd struct - to be alive (and not added to freelist) until the end of this function */ - REF_BY(fd, 1, reason); - /* Remove the fd from the polling island */ if (fd->pi != NULL) { - polling_island *pi = fd->pi; - polling_island_remove_fd(pi, fd, is_fd_closed, &error); + polling_island_remove_fd(fd->pi, fd, is_fd_closed, &error); unref_pi = fd->pi; fd->pi = NULL; } @@ -680,7 +635,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); gpr_mu_unlock(&fd->mu); - UNREF_BY(fd, 2, reason); /* Drop the reference */ + + /* We are done with this fd. Release it (i.e add back to freelist) */ + add_fd_to_freelist(fd); + if (unref_pi != NULL) { /* Unref stale polling island here, outside the fd lock above. The polling island owns a workqueue which owns an fd, and unreffing From aa033db15e3cc3bb43b8cca0df9edccbf24690fd Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 11:00:27 -0700 Subject: [PATCH 113/177] Rename polling_island to epoll_set --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 382 +++++++++--------- 1 file changed, 191 insertions(+), 191 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 96e8000da4e..448f2685fef 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -82,14 +82,14 @@ static int grpc_polling_trace = 0; /* Disabled by default */ * needed) */ static grpc_wakeup_fd global_wakeup_fd; -struct polling_island; +struct epoll_set; /******************************************************************************* * Fd Declarations */ struct grpc_fd { gpr_mu mu; - struct polling_island *pi; + struct epoll_set *eps; int fd; @@ -115,25 +115,25 @@ static void fd_global_shutdown(void); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) -#define PI_UNREF(exec_ctx, p, r) \ - pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) +#define EPS_ADD_REF(p, r) eps_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define EPS_UNREF(exec_ctx, p, r) \ + eps_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) #else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ -#define PI_ADD_REF(p, r) pi_add_ref((p)) -#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) +#define EPS_ADD_REF(p, r) eps_add_ref((p)) +#define EPS_UNREF(exec_ctx, p, r) eps_unref((exec_ctx), (p)) -#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ +#endif /* !defined(GRPC_EPS_REF_COUNT_DEBUG) */ /* This is also used as grpc_workqueue (by directly casting it) */ -typedef struct polling_island { +typedef struct epoll_set { grpc_closure_scheduler workqueue_scheduler; - /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + /* Ref count. Use EPS_ADD_REF() and EPS_UNREF() macros to increment/decrement the refcount. Once the ref count becomes zero, this structure is destroyed which means we should ensure that there is never a scenario where a - PI_ADD_REF() is racing with a PI_UNREF() that just made the ref_count + EPS_ADD_REF() is racing with a EPS_UNREF() that just made the ref_count zero. */ gpr_atm ref_count; @@ -154,7 +154,7 @@ typedef struct polling_island { /* The fd of the underlying epoll set */ int epoll_fd; -} polling_island; +} epoll_set; /******************************************************************************* * Pollset Declarations @@ -168,7 +168,7 @@ struct grpc_pollset_worker { struct grpc_pollset { gpr_mu mu; - struct polling_island *pi; + struct epoll_set *eps; grpc_pollset_worker root_worker; bool kicked_without_pollers; @@ -187,9 +187,9 @@ struct grpc_pollset_set {}; * Dedicated polling threads and pollsets - Declarations */ -size_t g_num_pi = 1; -struct polling_island **g_polling_islands = NULL; -size_t g_num_threads_per_pi = 1; +size_t g_num_eps = 1; +struct epoll_set **g_epoll_sets = NULL; +size_t g_num_threads_per_eps = 1; gpr_thd_id *g_poller_threads = NULL; /* Used as read-notifier pollsets for fds. We won't be using read notifier @@ -197,9 +197,9 @@ gpr_thd_id *g_poller_threads = NULL; * return */ grpc_pollset g_read_notifier; -static void add_fd_to_dedicated_pi(grpc_fd *fd); -static bool init_dedicated_polling_islands(); -static void shutdown_dedicated_polling_islands(); +static void add_fd_to_dedicated_eps(grpc_fd *fd); +static bool init_dedicated_epoll_sets(); +static void shutdown_dedicated_epoll_sets(); static void poller_thread_loop(void *arg); static void start_dedicated_poller_threads(); static void shutdown_dedicated_poller_threads(); @@ -229,14 +229,14 @@ static bool append_error(grpc_error **composite, grpc_error *error, NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ -static grpc_wakeup_fd polling_island_wakeup_fd; +static grpc_wakeup_fd epoll_set_wakeup_fd; /* The polling island being polled right now. See comments in workqueue_maybe_wakeup for why this is tracked. */ -static __thread polling_island *g_current_thread_polling_island; +static __thread epoll_set *g_current_thread_epoll_set; /* Forward declaration */ -static void polling_island_delete(polling_island *pi); +static void epoll_set_delete(epoll_set *eps); static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_error *error); @@ -254,31 +254,31 @@ gpr_atm g_epoll_sync; static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { workqueue_enqueue, workqueue_enqueue, "workqueue"}; -static void pi_add_ref(polling_island *pi); -static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void eps_add_ref(epoll_set *eps); +static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG -static void pi_add_ref_dbg(polling_island *pi, const char *reason, +static void eps_add_ref_dbg(epoll_set *eps, const char *reason, const char *file, int line) { - long old_cnt = gpr_atm_acq_load(&pi->ref_count); - pi_add_ref(pi); - gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", - (void *)pi, old_cnt, old_cnt + 1, reason, file, line); + long old_cnt = gpr_atm_acq_load(&eps->ref_count); + eps_add_ref(eps); + gpr_log(GPR_DEBUG, "Add ref eps: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)eps, old_cnt, old_cnt + 1, reason, file, line); } -static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, +static void eps_unref_dbg(grpc_exec_ctx *exec_ctx, epoll_set *eps, const char *reason, const char *file, int line) { - long old_cnt = gpr_atm_acq_load(&pi->ref_count); - pi_unref(exec_ctx, pi); - gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", - (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); + long old_cnt = gpr_atm_acq_load(&eps->ref_count); + eps_unref(exec_ctx, eps); + gpr_log(GPR_DEBUG, "Unref eps: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)eps, old_cnt, (old_cnt - 1), reason, file, line); } static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, const char *file, int line, const char *reason) { if (workqueue != NULL) { - pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + eps_add_ref_dbg((epoll_set *)workqueue, reason, file, line); } return workqueue; } @@ -286,13 +286,13 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, const char *file, int line, const char *reason) { if (workqueue != NULL) { - pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + eps_unref_dbg(exec_ctx, (epoll_set *)workqueue, reason, file, line); } } #else static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { if (workqueue != NULL) { - pi_add_ref((polling_island *)workqueue); + eps_add_ref((epoll_set *)workqueue); } return workqueue; } @@ -300,31 +300,31 @@ static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) { if (workqueue != NULL) { - pi_unref(exec_ctx, (polling_island *)workqueue); + eps_unref(exec_ctx, (epoll_set *)workqueue); } } #endif -static void pi_add_ref(polling_island *pi) { - gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +static void eps_add_ref(epoll_set *eps) { + gpr_atm_no_barrier_fetch_add(&eps->ref_count, 1); } -static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { +static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { /* If ref count went to zero, delete the polling island. This deletion is not done under a lock since once the ref count goes to zero, we are guaranteed that no one else holds a reference to the polling island (and - that there is no racing pi_add_ref() call either).*/ - if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { - polling_island_delete(pi); + that there is no racing eps_add_ref() call either).*/ + if (1 == gpr_atm_full_fetch_add(&eps->ref_count, -1)) { + epoll_set_delete(eps); } } -static void polling_island_add_fd_locked(polling_island *pi, grpc_fd *fd, +static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, grpc_error **error) { int err; struct epoll_event ev; char *err_msg; - const char *err_desc = "polling_island_add_fd_locked"; + const char *err_desc = "epoll_set_add_fd_locked"; #ifdef GRPC_TSAN /* See the definition of g_epoll_sync for more context */ @@ -333,55 +333,55 @@ static void polling_island_add_fd_locked(polling_island *pi, grpc_fd *fd, ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); ev.data.ptr = fd; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); if (err < 0 && errno != EEXIST) { gpr_asprintf( &err_msg, "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", - pi->epoll_fd, fd->fd, errno, strerror(errno)); + eps->epoll_fd, fd->fd, errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } } -static void polling_island_add_wakeup_fd_locked(polling_island *pi, +static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, grpc_wakeup_fd *wakeup_fd, grpc_error **error) { struct epoll_event ev; int err; char *err_msg; - const char *err_desc = "polling_island_add_wakeup_fd"; + const char *err_desc = "epoll_set_add_wakeup_fd"; ev.events = (uint32_t)(EPOLLIN | EPOLLET); ev.data.ptr = wakeup_fd; - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_ADD, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); if (err < 0 && errno != EEXIST) { gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } } -static void polling_island_remove_fd(polling_island *pi, grpc_fd *fd, +static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, bool is_fd_closed, grpc_error **error) { int err; char *err_msg; - const char *err_desc = "polling_island_remove_fd"; + const char *err_desc = "epoll_set_remove_fd"; /* If fd is already closed, then it would have been automatically been removed from the epoll set */ if (!is_fd_closed) { - err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + err = epoll_ctl(eps->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); if (err < 0 && errno != ENOENT) { gpr_asprintf( &err_msg, "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", - pi->epoll_fd, fd->fd, errno, strerror(errno)); + eps->epoll_fd, fd->fd, errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -389,74 +389,74 @@ static void polling_island_remove_fd(polling_island *pi, grpc_fd *fd, } /* Might return NULL in case of an error */ -static polling_island *polling_island_create(grpc_error **error) { - polling_island *pi = NULL; - const char *err_desc = "polling_island_create"; +static epoll_set *epoll_set_create(grpc_error **error) { + epoll_set *eps = NULL; + const char *err_desc = "epoll_set_create"; *error = GRPC_ERROR_NONE; - pi = gpr_malloc(sizeof(*pi)); - pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; - pi->epoll_fd = -1; + eps = gpr_malloc(sizeof(*eps)); + eps->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + eps->epoll_fd = -1; - gpr_mu_init(&pi->workqueue_read_mu); - gpr_mpscq_init(&pi->workqueue_items); - gpr_atm_rel_store(&pi->workqueue_item_count, 0); + gpr_mu_init(&eps->workqueue_read_mu); + gpr_mpscq_init(&eps->workqueue_items); + gpr_atm_rel_store(&eps->workqueue_item_count, 0); - gpr_atm_rel_store(&pi->ref_count, 0); - gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&eps->ref_count, 0); + gpr_atm_rel_store(&eps->poller_count, 0); - gpr_atm_rel_store(&pi->is_shutdown, false); + gpr_atm_rel_store(&eps->is_shutdown, false); - if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + if (!append_error(error, grpc_wakeup_fd_init(&eps->workqueue_wakeup_fd), err_desc)) { goto done; } - pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + eps->epoll_fd = epoll_create1(EPOLL_CLOEXEC); - if (pi->epoll_fd < 0) { + if (eps->epoll_fd < 0) { append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); goto done; } - polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); - polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + epoll_set_add_wakeup_fd_locked(eps, &global_wakeup_fd, error); + epoll_set_add_wakeup_fd_locked(eps, &eps->workqueue_wakeup_fd, error); done: if (*error != GRPC_ERROR_NONE) { - polling_island_delete(pi); - pi = NULL; + epoll_set_delete(eps); + eps = NULL; } - return pi; + return eps; } -static void polling_island_delete(polling_island *pi) { - if (pi->epoll_fd >= 0) { - close(pi->epoll_fd); +static void epoll_set_delete(epoll_set *eps) { + if (eps->epoll_fd >= 0) { + close(eps->epoll_fd); } - GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); - gpr_mu_destroy(&pi->workqueue_read_mu); - gpr_mpscq_destroy(&pi->workqueue_items); - grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + GPR_ASSERT(gpr_atm_no_barrier_load(&eps->workqueue_item_count) == 0); + gpr_mu_destroy(&eps->workqueue_read_mu); + gpr_mpscq_destroy(&eps->workqueue_items); + grpc_wakeup_fd_destroy(&eps->workqueue_wakeup_fd); - gpr_free(pi); + gpr_free(eps); } -static void workqueue_maybe_wakeup(polling_island *pi) { +static void workqueue_maybe_wakeup(epoll_set *eps) { /* If this thread is the current poller, then it may be that it's about to decrement the current poller count, so we need to look past this thread */ - bool is_current_poller = (g_current_thread_polling_island == pi); + bool is_current_poller = (g_current_thread_epoll_set == eps); gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; - gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + gpr_atm current_pollers = gpr_atm_no_barrier_load(&eps->poller_count); /* Only issue a wakeup if it's likely that some poller could come in and take it right now. Note that since we do an anticipatory mpscq_pop every poll loop, it's ok if we miss the wakeup here, as we'll get the work item when the next poller enters anyway. */ if (current_pollers > min_current_pollers_for_wakeup) { GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", - grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + grpc_wakeup_fd_wakeup(&eps->workqueue_wakeup_fd)); } } @@ -468,12 +468,12 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, * this kicks off ends up destroying the workqueue before this function * completes */ GRPC_WORKQUEUE_REF(workqueue, "enqueue"); - polling_island *pi = (polling_island *)workqueue; - gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + epoll_set *eps = (epoll_set *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&eps->workqueue_item_count, 1); closure->error_data.error = error; - gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + gpr_mpscq_push(&eps->workqueue_items, &closure->next_data.atm_next); if (last == 0) { - workqueue_maybe_wakeup(pi); + workqueue_maybe_wakeup(eps); } GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); @@ -481,24 +481,24 @@ static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, } static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { - polling_island *pi = (polling_island *)workqueue; + epoll_set *eps = (epoll_set *)workqueue; return workqueue == NULL ? grpc_schedule_on_exec_ctx - : &pi->workqueue_scheduler; + : &eps->workqueue_scheduler; } -static grpc_error *polling_island_global_init() { +static grpc_error *epoll_set_global_init() { grpc_error *error = GRPC_ERROR_NONE; - error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + error = grpc_wakeup_fd_init(&epoll_set_wakeup_fd); if (error == GRPC_ERROR_NONE) { - error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + error = grpc_wakeup_fd_wakeup(&epoll_set_wakeup_fd); } return error; } -static void polling_island_global_shutdown() { - grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +static void epoll_set_global_shutdown() { + grpc_wakeup_fd_destroy(&epoll_set_wakeup_fd); } /******************************************************************************* @@ -570,7 +570,7 @@ static grpc_fd *fd_create(int fd, const char *name) { * is a newly created fd (or an fd we got from the freelist), no one else * would be holding a lock to it anyway. */ gpr_mu_lock(&new_fd->mu); - new_fd->pi = NULL; + new_fd->eps = NULL; new_fd->fd = fd; new_fd->orphaned = false; @@ -588,8 +588,8 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); gpr_free(fd_name); - /* Associate the fd with one of the dedicated pi */ - add_fd_to_dedicated_pi(new_fd); + /* Associate the fd with one of the dedicated eps */ + add_fd_to_dedicated_eps(new_fd); return new_fd; } @@ -609,7 +609,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, const char *reason) { bool is_fd_closed = false; grpc_error *error = GRPC_ERROR_NONE; - polling_island *unref_pi = NULL; + epoll_set *unref_eps = NULL; gpr_mu_lock(&fd->mu); fd->on_done_closure = on_done; @@ -626,10 +626,10 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; /* Remove the fd from the polling island */ - if (fd->pi != NULL) { - polling_island_remove_fd(fd->pi, fd, is_fd_closed, &error); - unref_pi = fd->pi; - fd->pi = NULL; + if (fd->eps != NULL) { + epoll_set_remove_fd(fd->eps, fd, is_fd_closed, &error); + unref_eps = fd->eps; + fd->eps = NULL; } grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); @@ -639,12 +639,12 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, /* We are done with this fd. Release it (i.e add back to freelist) */ add_fd_to_freelist(fd); - if (unref_pi != NULL) { + if (unref_eps != NULL) { /* Unref stale polling island here, outside the fd lock above. The polling island owns a workqueue which owns an fd, and unreffing inside the lock can cause an eventual lock loop that makes TSAN very unhappy. */ - PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + EPS_UNREF(exec_ctx, unref_eps, "fd_orphan"); } GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); GRPC_ERROR_UNREF(error); @@ -805,7 +805,7 @@ static grpc_error *kick_poller(void) { static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; - pollset->pi = NULL; + pollset->eps = NULL; pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; pollset->kicked_without_pollers = false; @@ -823,12 +823,12 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, +static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, char *reason) { - if (ps->pi != NULL) { - PI_UNREF(exec_ctx, ps->pi, reason); + if (ps->eps != NULL) { + EPS_UNREF(exec_ctx, ps->eps, reason); } - ps->pi = NULL; + ps->eps = NULL; } static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, @@ -838,8 +838,8 @@ static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, pollset->finish_shutdown_called = true; - /* Release the ref and set pollset->pi to NULL */ - pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + /* Release the ref and set pollset->eps to NULL */ + pollset_release_epoll_set(exec_ctx, pollset, "ps_shutdown"); grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); } @@ -872,13 +872,13 @@ static void pollset_destroy(grpc_pollset *pollset) { } static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, - polling_island *pi) { - if (gpr_mu_trylock(&pi->workqueue_read_mu)) { - gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); - gpr_mu_unlock(&pi->workqueue_read_mu); + epoll_set *eps) { + if (gpr_mu_trylock(&eps->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&eps->workqueue_items); + gpr_mu_unlock(&eps->workqueue_read_mu); if (n != NULL) { - if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { - workqueue_maybe_wakeup(pi); + if (gpr_atm_full_fetch_add(&eps->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(eps); } grpc_closure *c = (grpc_closure *)n; grpc_error *error = c->error_data.error; @@ -888,11 +888,11 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, c->cb(exec_ctx, c->cb_arg, error); GRPC_ERROR_UNREF(error); return true; - } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + } else if (gpr_atm_no_barrier_load(&eps->workqueue_item_count) > 0) { /* n == NULL might mean there's work but it's not available to be popped * yet - try to ensure another workqueue wakes up to check shortly if so */ - workqueue_maybe_wakeup(pi); + workqueue_maybe_wakeup(eps); } } return false; @@ -900,7 +900,7 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, #define GRPC_EPOLL_MAX_EVENTS 100 static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, - polling_island *pi, grpc_error **error) { + epoll_set *eps, grpc_error **error) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; @@ -930,13 +930,13 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, grpc_timer_consume_kick(); append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { + } else if (data_ptr == &eps->workqueue_wakeup_fd) { append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + grpc_wakeup_fd_consume_wakeup(&eps->workqueue_wakeup_fd), err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - gpr_atm_rel_store(&pi->is_shutdown, 1); + maybe_do_workqueue_work(exec_ctx, eps); + } else if (data_ptr == &epoll_set_wakeup_fd) { + gpr_atm_rel_store(&eps->is_shutdown, 1); gpr_log(GPR_INFO, "pollset poller: shutdown set"); } else { grpc_fd *fd = data_ptr; @@ -953,41 +953,41 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, } } -static void polling_island_work(grpc_exec_ctx *exec_ctx, polling_island *pi, +static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, grpc_error **error) { int epoll_fd = -1; - GPR_TIMER_BEGIN("polling_island_work", 0); + GPR_TIMER_BEGIN("epoll_set_work", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the polling island. */ - epoll_fd = pi->epoll_fd; + epoll_fd = eps->epoll_fd; /* Add an extra ref so that the island does not get destroyed (which means the epoll_fd won't be closed) while we are are doing an epoll_wait() on the epoll_fd */ - PI_ADD_REF(pi, "ps_work"); + EPS_ADD_REF(eps, "ps_work"); /* If we get some workqueue work to do, it might end up completing an item on the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ - if (!maybe_do_workqueue_work(exec_ctx, pi)) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); - g_current_thread_polling_island = pi; + if (!maybe_do_workqueue_work(exec_ctx, eps)) { + gpr_atm_no_barrier_fetch_add(&eps->poller_count, 1); + g_current_thread_epoll_set = eps; - do_epoll_wait(exec_ctx, epoll_fd, pi, error); + do_epoll_wait(exec_ctx, epoll_fd, eps, error); - g_current_thread_polling_island = NULL; - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + g_current_thread_epoll_set = NULL; + gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); } /* Before leaving, release the extra ref we added to the polling island. It - is important to use "pi" here (i.e our old copy of pollset->pi + is important to use "eps" here (i.e our old copy of pollset->eps that we got before releasing the polling island lock). This is because - pollset->pi pointer might get udpated in other parts of the + pollset->eps pointer might get udpated in other parts of the code when there is an island merge while we are doing epoll_wait() above */ - PI_UNREF(exec_ctx, pi, "ps_work"); + EPS_UNREF(exec_ctx, eps, "ps_work"); - GPR_TIMER_END("polling_island_work", 0); + GPR_TIMER_END("epoll_set_work", 0); } /* pollset->mu lock must be held by the caller before calling this. @@ -1110,10 +1110,10 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, static void shutdown_engine(void) { shutdown_dedicated_poller_threads(); - shutdown_dedicated_polling_islands(); + shutdown_dedicated_epoll_sets(); fd_global_shutdown(); pollset_global_shutdown(); - polling_island_global_shutdown(); + epoll_set_global_shutdown(); gpr_log(GPR_INFO, "ev-epoll-threadpool engine shutdown complete"); } @@ -1158,13 +1158,13 @@ static const grpc_event_engine_vtable vtable = { /***************************************************************************** * Dedicated polling threads and pollsets - Definitions */ -static void add_fd_to_dedicated_pi(grpc_fd *fd) { - GPR_ASSERT(fd->pi == NULL); - GPR_TIMER_BEGIN("add_fd_to_dedicated_pi", 0); +static void add_fd_to_dedicated_eps(grpc_fd *fd) { + GPR_ASSERT(fd->eps == NULL); + GPR_TIMER_BEGIN("add_fd_to_dedicated_eps", 0); grpc_error *error = GRPC_ERROR_NONE; - size_t idx = ((size_t)rand()) % g_num_pi; - polling_island *pi = g_polling_islands[idx]; + size_t idx = ((size_t)rand()) % g_num_eps; + epoll_set *eps = g_epoll_sets[idx]; gpr_mu_lock(&fd->mu); @@ -1173,70 +1173,70 @@ static void add_fd_to_dedicated_pi(grpc_fd *fd) { return; /* Early out */ } - polling_island_add_fd_locked(pi, fd, &error); - PI_ADD_REF(pi, "fd"); - fd->pi = pi; + epoll_set_add_fd_locked(eps, fd, &error); + EPS_ADD_REF(eps, "fd"); + fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_dedicated_pi (fd: %d, pi idx = %ld)", fd->fd, + GRPC_POLLING_TRACE("add_fd_to_dedicated_eps (fd: %d, eps idx = %ld)", fd->fd, idx); gpr_mu_unlock(&fd->mu); - GRPC_LOG_IF_ERROR("add_fd_to_dedicated_pi", error); - GPR_TIMER_END("add_fd_to_dedicated_pi", 0); + GRPC_LOG_IF_ERROR("add_fd_to_dedicated_eps", error); + GPR_TIMER_END("add_fd_to_dedicated_eps", 0); } -static bool init_dedicated_polling_islands() { +static bool init_dedicated_epoll_sets() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; - g_polling_islands = - (polling_island **)malloc(g_num_pi * sizeof(polling_island *)); + g_epoll_sets = + (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); - for (size_t i = 0; i < g_num_pi; i++) { - g_polling_islands[i] = polling_island_create(&error); - if (g_polling_islands[i] == NULL) { + for (size_t i = 0; i < g_num_eps; i++) { + g_epoll_sets[i] = epoll_set_create(&error); + if (g_epoll_sets[i] == NULL) { gpr_log(GPR_ERROR, "Error in creating a dedicated polling island"); - g_num_pi = i; /* Helps cleanup */ - shutdown_dedicated_polling_islands(); + g_num_eps = i; /* Helps cleanup */ + shutdown_dedicated_epoll_sets(); is_success = false; goto done; } - PI_ADD_REF(g_polling_islands[i], "init_dedicated_polling_islands"); + EPS_ADD_REF(g_epoll_sets[i], "init_dedicated_epoll_sets"); } gpr_mu *mu; pollset_init(&g_read_notifier, &mu); done: - GRPC_LOG_IF_ERROR("init_dedicated_polling_islands", error); + GRPC_LOG_IF_ERROR("init_dedicated_epoll_sets", error); return is_success; } -static void shutdown_dedicated_polling_islands() { - if (!g_polling_islands) { +static void shutdown_dedicated_epoll_sets() { + if (!g_epoll_sets) { return; } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - for (size_t i = 0; i < g_num_pi; i++) { - PI_UNREF(&exec_ctx, g_polling_islands[i], - "shutdown_dedicated_polling_islands"); + for (size_t i = 0; i < g_num_eps; i++) { + EPS_UNREF(&exec_ctx, g_epoll_sets[i], + "shutdown_dedicated_epoll_sets"); } grpc_exec_ctx_finish(&exec_ctx); - gpr_free(g_polling_islands); - g_polling_islands = NULL; + gpr_free(g_epoll_sets); + g_epoll_sets = NULL; pollset_destroy(&g_read_notifier); } static void poller_thread_loop(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_error *error = GRPC_ERROR_NONE; - polling_island *pi = (polling_island *)arg; + epoll_set *eps = (epoll_set *)arg; - while (!gpr_atm_acq_load(&pi->is_shutdown)) { - polling_island_work(&exec_ctx, pi, &error); + while (!gpr_atm_acq_load(&eps->is_shutdown)) { + epoll_set_work(&exec_ctx, eps, &error); grpc_exec_ctx_flush(&exec_ctx); } @@ -1244,37 +1244,37 @@ static void poller_thread_loop(void *arg) { GRPC_LOG_IF_ERROR("poller_thread_loop", error); } -/* g_polling_islands MUST be initialized before calling this */ +/* g_epoll_sets MUST be initialized before calling this */ static void start_dedicated_poller_threads() { - GPR_ASSERT(g_polling_islands); + GPR_ASSERT(g_epoll_sets); gpr_log(GPR_INFO, "Starting poller threads"); /* One thread per pollset */ - g_poller_threads = (gpr_thd_id *)malloc(g_num_pi * sizeof(gpr_thd_id)); + g_poller_threads = (gpr_thd_id *)malloc(g_num_eps * sizeof(gpr_thd_id)); gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); - for (size_t i = 0; i < g_num_pi; i++) { + for (size_t i = 0; i < g_num_eps; i++) { gpr_thd_new(&g_poller_threads[i], poller_thread_loop, - (void *)g_polling_islands[i], &options); + (void *)g_epoll_sets[i], &options); } } static void shutdown_dedicated_poller_threads() { GPR_ASSERT(g_poller_threads); - GPR_ASSERT(g_polling_islands); + GPR_ASSERT(g_epoll_sets); grpc_error *error = GRPC_ERROR_NONE; gpr_log(GPR_INFO, "Shutting down pollers"); - polling_island *pi = NULL; - for (size_t i = 0; i < g_num_pi; i++) { - pi = g_polling_islands[i]; - polling_island_add_wakeup_fd_locked(pi, &polling_island_wakeup_fd, &error); + epoll_set *eps = NULL; + for (size_t i = 0; i < g_num_eps; i++) { + eps = g_epoll_sets[i]; + epoll_set_add_wakeup_fd_locked(eps, &epoll_set_wakeup_fd, &error); } - for (size_t i = 0; i < g_num_pi; i++) { + for (size_t i = 0; i < g_num_eps; i++) { gpr_thd_join(g_poller_threads[i]); } @@ -1316,12 +1316,12 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!GRPC_LOG_IF_ERROR("polling_island_global_init", - polling_island_global_init())) { + if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", + epoll_set_global_init())) { return NULL; } - if (!init_dedicated_polling_islands()) { + if (!init_dedicated_epoll_sets()) { return NULL; } From 0a3a416dde6b6837900e4086c9366da11ea514b0 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 11:00:53 -0700 Subject: [PATCH 114/177] Format --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 448f2685fef..d3e7820149e 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -259,7 +259,7 @@ static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps); #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG static void eps_add_ref_dbg(epoll_set *eps, const char *reason, - const char *file, int line) { + const char *file, int line) { long old_cnt = gpr_atm_acq_load(&eps->ref_count); eps_add_ref(eps); gpr_log(GPR_DEBUG, "Add ref eps: %p, old: %ld -> new:%ld (%s) - (%s, %d)", @@ -267,7 +267,7 @@ static void eps_add_ref_dbg(epoll_set *eps, const char *reason, } static void eps_unref_dbg(grpc_exec_ctx *exec_ctx, epoll_set *eps, - const char *reason, const char *file, int line) { + const char *reason, const char *file, int line) { long old_cnt = gpr_atm_acq_load(&eps->ref_count); eps_unref(exec_ctx, eps); gpr_log(GPR_DEBUG, "Unref eps: %p, old:%ld -> new:%ld (%s) - (%s, %d)", @@ -320,7 +320,7 @@ static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { } static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, - grpc_error **error) { + grpc_error **error) { int err; struct epoll_event ev; char *err_msg; @@ -345,8 +345,8 @@ static void epoll_set_add_fd_locked(epoll_set *eps, grpc_fd *fd, } static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, - grpc_wakeup_fd *wakeup_fd, - grpc_error **error) { + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { struct epoll_event ev; int err; char *err_msg; @@ -367,8 +367,8 @@ static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, } } -static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, - bool is_fd_closed, grpc_error **error) { +static void epoll_set_remove_fd(epoll_set *eps, grpc_fd *fd, bool is_fd_closed, + grpc_error **error) { int err; char *err_msg; const char *err_desc = "epoll_set_remove_fd"; @@ -823,8 +823,8 @@ static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { grpc_lfev_set_ready(exec_ctx, &fd->write_closure); } -static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, - grpc_pollset *ps, char *reason) { +static void pollset_release_epoll_set(grpc_exec_ctx *exec_ctx, grpc_pollset *ps, + char *reason) { if (ps->eps != NULL) { EPS_UNREF(exec_ctx, ps->eps, reason); } @@ -871,8 +871,7 @@ static void pollset_destroy(grpc_pollset *pollset) { gpr_mu_destroy(&pollset->mu); } -static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, - epoll_set *eps) { +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, epoll_set *eps) { if (gpr_mu_trylock(&eps->workqueue_read_mu)) { gpr_mpscq_node *n = gpr_mpscq_pop(&eps->workqueue_items); gpr_mu_unlock(&eps->workqueue_read_mu); @@ -899,8 +898,8 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, } #define GRPC_EPOLL_MAX_EVENTS 100 -static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, - epoll_set *eps, grpc_error **error) { +static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, + grpc_error **error) { struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int ep_rv; char *err_msg; @@ -954,7 +953,7 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, } static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, - grpc_error **error) { + grpc_error **error) { int epoll_fd = -1; GPR_TIMER_BEGIN("epoll_set_work", 0); @@ -1189,8 +1188,7 @@ static bool init_dedicated_epoll_sets() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; - g_epoll_sets = - (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); + g_epoll_sets = (epoll_set **)malloc(g_num_eps * sizeof(epoll_set *)); for (size_t i = 0; i < g_num_eps; i++) { g_epoll_sets[i] = epoll_set_create(&error); @@ -1220,8 +1218,7 @@ static void shutdown_dedicated_epoll_sets() { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; for (size_t i = 0; i < g_num_eps; i++) { - EPS_UNREF(&exec_ctx, g_epoll_sets[i], - "shutdown_dedicated_epoll_sets"); + EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_dedicated_epoll_sets"); } grpc_exec_ctx_finish(&exec_ctx); @@ -1316,8 +1313,7 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", - epoll_set_global_init())) { + if (!GRPC_LOG_IF_ERROR("epoll_set_global_init", epoll_set_global_init())) { return NULL; } From c5cae77ef5f3fa0d2401cb648020a35295b9c8b5 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 11:39:03 -0700 Subject: [PATCH 115/177] Rename a few functions/variables --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 82 +++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index d3e7820149e..d11dbbfae8c 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -149,7 +149,7 @@ typedef struct epoll_set { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; - /* Is the polling island shutdown */ + /* Is the epoll set shutdown */ gpr_atm is_shutdown; /* The fd of the underlying epoll set */ @@ -197,12 +197,12 @@ gpr_thd_id *g_poller_threads = NULL; * return */ grpc_pollset g_read_notifier; -static void add_fd_to_dedicated_eps(grpc_fd *fd); -static bool init_dedicated_epoll_sets(); -static void shutdown_dedicated_epoll_sets(); +static void add_fd_to_eps(grpc_fd *fd); +static bool init_epoll_sets(); +static void shutdown_epoll_sets(); static void poller_thread_loop(void *arg); -static void start_dedicated_poller_threads(); -static void shutdown_dedicated_poller_threads(); +static void start_poller_threads(); +static void shutdown_poller_threads(); /******************************************************************************* * Common helpers @@ -223,15 +223,15 @@ static bool append_error(grpc_error **composite, grpc_error *error, */ /* The wakeup fd that is used to wake up all threads in a Polling island. This - is useful in the polling island merge operation where we need to wakeup all - the threads currently polling the smaller polling island (so that they can - start polling the new/merged polling island) + is useful in the epoll set merge operation where we need to wakeup all + the threads currently polling the smaller epoll set (so that they can + start polling the new/merged epoll set) NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ static grpc_wakeup_fd epoll_set_wakeup_fd; -/* The polling island being polled right now. +/* The epoll set being polled right now. See comments in workqueue_maybe_wakeup for why this is tracked. */ static __thread epoll_set *g_current_thread_epoll_set; @@ -310,9 +310,9 @@ static void eps_add_ref(epoll_set *eps) { } static void eps_unref(grpc_exec_ctx *exec_ctx, epoll_set *eps) { - /* If ref count went to zero, delete the polling island. This deletion is + /* If ref count went to zero, delete the epoll set. This deletion is not done under a lock since once the ref count goes to zero, we are - guaranteed that no one else holds a reference to the polling island (and + guaranteed that no one else holds a reference to the epoll set (and that there is no racing eps_add_ref() call either).*/ if (1 == gpr_atm_full_fetch_add(&eps->ref_count, -1)) { epoll_set_delete(eps); @@ -588,8 +588,8 @@ static grpc_fd *fd_create(int fd, const char *name) { gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); gpr_free(fd_name); - /* Associate the fd with one of the dedicated eps */ - add_fd_to_dedicated_eps(new_fd); + /* Associate the fd with one of the eps */ + add_fd_to_eps(new_fd); return new_fd; } @@ -625,7 +625,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, fd->orphaned = true; - /* Remove the fd from the polling island */ + /* Remove the fd from the epoll set */ if (fd->eps != NULL) { epoll_set_remove_fd(fd->eps, fd, is_fd_closed, &error); unref_eps = fd->eps; @@ -640,8 +640,8 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, add_fd_to_freelist(fd); if (unref_eps != NULL) { - /* Unref stale polling island here, outside the fd lock above. - The polling island owns a workqueue which owns an fd, and unreffing + /* Unref stale epoll set here, outside the fd lock above. + The epoll set owns a workqueue which owns an fd, and unreffing inside the lock can cause an eventual lock loop that makes TSAN very unhappy. */ EPS_UNREF(exec_ctx, unref_eps, "fd_orphan"); @@ -853,7 +853,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); /* If the pollset has any workers, we cannot call finish_shutdown_locked() - because it would release the underlying polling island. In such a case, we + because it would release the underlying epoll set. In such a case, we let the last worker call finish_shutdown_locked() from pollset_work() */ if (!pollset_has_workers(pollset)) { GPR_ASSERT(!pollset->finish_shutdown_called); @@ -958,7 +958,7 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, GPR_TIMER_BEGIN("epoll_set_work", 0); /* Since epoll_fd is immutable, it is safe to read it without a lock on the - polling island. */ + epoll set. */ epoll_fd = eps->epoll_fd; /* Add an extra ref so that the island does not get destroyed (which means @@ -979,9 +979,9 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); } - /* Before leaving, release the extra ref we added to the polling island. It + /* Before leaving, release the extra ref we added to the epoll set. It is important to use "eps" here (i.e our old copy of pollset->eps - that we got before releasing the polling island lock). This is because + that we got before releasing the epoll set lock). This is because pollset->eps pointer might get udpated in other parts of the code when there is an island merge while we are doing epoll_wait() above */ EPS_UNREF(exec_ctx, eps, "ps_work"); @@ -1108,8 +1108,8 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, */ static void shutdown_engine(void) { - shutdown_dedicated_poller_threads(); - shutdown_dedicated_epoll_sets(); + shutdown_poller_threads(); + shutdown_epoll_sets(); fd_global_shutdown(); pollset_global_shutdown(); epoll_set_global_shutdown(); @@ -1157,9 +1157,9 @@ static const grpc_event_engine_vtable vtable = { /***************************************************************************** * Dedicated polling threads and pollsets - Definitions */ -static void add_fd_to_dedicated_eps(grpc_fd *fd) { +static void add_fd_to_eps(grpc_fd *fd) { GPR_ASSERT(fd->eps == NULL); - GPR_TIMER_BEGIN("add_fd_to_dedicated_eps", 0); + GPR_TIMER_BEGIN("add_fd_to_eps", 0); grpc_error *error = GRPC_ERROR_NONE; size_t idx = ((size_t)rand()) % g_num_eps; @@ -1176,15 +1176,15 @@ static void add_fd_to_dedicated_eps(grpc_fd *fd) { EPS_ADD_REF(eps, "fd"); fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_dedicated_eps (fd: %d, eps idx = %ld)", fd->fd, + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, idx); gpr_mu_unlock(&fd->mu); - GRPC_LOG_IF_ERROR("add_fd_to_dedicated_eps", error); - GPR_TIMER_END("add_fd_to_dedicated_eps", 0); + GRPC_LOG_IF_ERROR("add_fd_to_eps", error); + GPR_TIMER_END("add_fd_to_eps", 0); } -static bool init_dedicated_epoll_sets() { +static bool init_epoll_sets() { grpc_error *error = GRPC_ERROR_NONE; bool is_success = true; @@ -1193,32 +1193,32 @@ static bool init_dedicated_epoll_sets() { for (size_t i = 0; i < g_num_eps; i++) { g_epoll_sets[i] = epoll_set_create(&error); if (g_epoll_sets[i] == NULL) { - gpr_log(GPR_ERROR, "Error in creating a dedicated polling island"); + gpr_log(GPR_ERROR, "Error in creating a epoll set"); g_num_eps = i; /* Helps cleanup */ - shutdown_dedicated_epoll_sets(); + shutdown_epoll_sets(); is_success = false; goto done; } - EPS_ADD_REF(g_epoll_sets[i], "init_dedicated_epoll_sets"); + EPS_ADD_REF(g_epoll_sets[i], "init_epoll_sets"); } gpr_mu *mu; pollset_init(&g_read_notifier, &mu); done: - GRPC_LOG_IF_ERROR("init_dedicated_epoll_sets", error); + GRPC_LOG_IF_ERROR("init_epoll_sets", error); return is_success; } -static void shutdown_dedicated_epoll_sets() { +static void shutdown_epoll_sets() { if (!g_epoll_sets) { return; } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; for (size_t i = 0; i < g_num_eps; i++) { - EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_dedicated_epoll_sets"); + EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_epoll_sets"); } grpc_exec_ctx_finish(&exec_ctx); @@ -1242,7 +1242,7 @@ static void poller_thread_loop(void *arg) { } /* g_epoll_sets MUST be initialized before calling this */ -static void start_dedicated_poller_threads() { +static void start_poller_threads() { GPR_ASSERT(g_epoll_sets); gpr_log(GPR_INFO, "Starting poller threads"); @@ -1258,7 +1258,7 @@ static void start_dedicated_poller_threads() { } } -static void shutdown_dedicated_poller_threads() { +static void shutdown_poller_threads() { GPR_ASSERT(g_poller_threads); GPR_ASSERT(g_epoll_sets); grpc_error *error = GRPC_ERROR_NONE; @@ -1275,8 +1275,8 @@ static void shutdown_dedicated_poller_threads() { gpr_thd_join(g_poller_threads[i]); } - gpr_log(GPR_ERROR, "polling island delete called"); - GRPC_LOG_IF_ERROR("shutdown_dedicated_poller_threads", error); + gpr_log(GPR_ERROR, "epoll set delete called"); + GRPC_LOG_IF_ERROR("shutdown_poller_threads", error); gpr_free(g_poller_threads); g_poller_threads = NULL; } @@ -1317,14 +1317,14 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { return NULL; } - if (!init_dedicated_epoll_sets()) { + if (!init_epoll_sets()) { return NULL; } /* TODO (sreek): Maynot be a good idea to start threads here (especially if * this engine doesn't get picked. Consider introducing an engine_init * function in the vtable */ - start_dedicated_poller_threads(); + start_poller_threads(); return &vtable; } From 84f75d448e2eb9c76301598cfaf8630603ef4a98 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 3 May 2017 13:06:35 -0700 Subject: [PATCH 116/177] Fix ASAN/TSAN failures - trace system is now thread safe when run with TSAN - fix a race in client_auth_filter.c - allow timer manager to run in single threaded mode for fuzzers --- .../client_channel/channel_connectivity.c | 2 +- .../client_channel/lb_policy/grpclb/grpclb.c | 40 ++++++------- .../lb_policy/round_robin/round_robin.c | 22 ++++---- .../ext/filters/http/http_filters_plugin.c | 1 + .../message_compress_filter.c | 5 +- .../message_compress_filter.h | 2 - .../chttp2/transport/chttp2_transport.c | 22 ++++---- .../chttp2/transport/chttp2_transport.h | 5 +- .../chttp2/transport/frame_settings.c | 6 +- .../transport/chttp2/transport/hpack_parser.c | 6 +- .../ext/transport/chttp2/transport/internal.h | 18 +++--- .../ext/transport/chttp2/transport/parsing.c | 8 +-- .../ext/transport/chttp2/transport/writing.c | 9 ++- src/core/lib/channel/channel_stack.c | 2 +- src/core/lib/channel/channel_stack.h | 4 +- src/core/lib/channel/channel_stack_builder.c | 2 +- src/core/lib/channel/channel_stack_builder.h | 2 +- src/core/lib/debug/trace.c | 16 ++++-- src/core/lib/debug/trace.h | 26 ++++++++- src/core/lib/http/parser.c | 4 +- src/core/lib/http/parser.h | 3 +- src/core/lib/iomgr/combiner.c | 12 ++-- src/core/lib/iomgr/combiner.h | 3 +- src/core/lib/iomgr/iomgr.c | 3 + src/core/lib/iomgr/iomgr.h | 3 + src/core/lib/iomgr/resource_quota.c | 16 +++--- src/core/lib/iomgr/resource_quota.h | 3 +- src/core/lib/iomgr/tcp_client_posix.c | 8 +-- src/core/lib/iomgr/tcp_posix.c | 14 ++--- src/core/lib/iomgr/tcp_posix.h | 3 +- src/core/lib/iomgr/tcp_server_posix.c | 2 +- src/core/lib/iomgr/timer_generic.c | 49 ++++++++-------- src/core/lib/iomgr/timer_manager.c | 56 +++++++++++++++---- src/core/lib/iomgr/timer_manager.h | 9 +++ .../credentials/jwt/jwt_credentials.c | 2 +- .../credentials/oauth2/oauth2_credentials.c | 2 +- .../security/transport/client_auth_filter.c | 2 +- .../lib/security/transport/secure_endpoint.c | 6 +- .../lib/security/transport/secure_endpoint.h | 2 +- src/core/lib/surface/api_trace.c | 2 +- src/core/lib/surface/api_trace.h | 4 +- src/core/lib/surface/call.c | 11 ++-- src/core/lib/surface/call.h | 6 +- src/core/lib/surface/completion_queue.c | 33 ++++++----- src/core/lib/surface/completion_queue.h | 9 +-- src/core/lib/surface/init.c | 3 +- src/core/lib/surface/server.c | 4 +- src/core/lib/surface/server.h | 3 +- src/core/lib/transport/bdp_estimator.c | 10 ++-- src/core/lib/transport/bdp_estimator.h | 3 +- src/core/lib/transport/connectivity_state.c | 12 ++-- src/core/lib/transport/connectivity_state.h | 3 +- src/core/tsi/fake_transport_security.c | 8 +-- src/core/tsi/ssl_transport_security.c | 2 +- src/core/tsi/transport_security.c | 2 +- src/core/tsi/transport_security.h | 3 +- src/core/tsi/transport_security_interface.h | 5 +- test/core/end2end/fuzzers/api_fuzzer.c | 4 ++ test/core/transport/connectivity_state_test.c | 2 +- 59 files changed, 312 insertions(+), 217 deletions(-) diff --git a/src/core/ext/filters/client_channel/channel_connectivity.c b/src/core/ext/filters/client_channel/channel_connectivity.c index 62f58fb278a..f83670db82e 100644 --- a/src/core/ext/filters/client_channel/channel_connectivity.c +++ b/src/core/ext/filters/client_channel/channel_connectivity.c @@ -132,7 +132,7 @@ static void partly_done(grpc_exec_ctx *exec_ctx, state_watcher *w, gpr_mu_lock(&w->mu); if (due_to_completion) { - if (grpc_trace_operation_failures) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures)) { GRPC_LOG_IF_ERROR("watch_completion_error", GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 37468101f0a..a15476dc23e 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -138,7 +138,7 @@ #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -int grpc_lb_glb_trace = 0; +grpc_tracer_flag grpc_lb_glb_trace; /* add lb_token of selected subchannel (address) to the call's initial * metadata */ @@ -224,7 +224,7 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg, } else { grpc_grpclb_client_stats_unref(wc_arg->client_stats); } - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Unreffing RR %p", (void *)wc_arg->rr_policy); } GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "wrapped_rr_closure"); @@ -575,7 +575,7 @@ static bool update_lb_connectivity_status_locked( GPR_ASSERT(new_rr_state_error == GRPC_ERROR_NONE); } - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Setting grpclb's state to %s from new RR policy %p state.", grpc_connectivity_state_name(new_rr_state), @@ -600,7 +600,7 @@ static bool pick_from_internal_rr_locked( (void **)&wc_arg->lb_token, &wc_arg->wrapper_closure); if (pick_done) { /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */ - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Unreffing RR (0x%" PRIxPTR ")", (intptr_t)wc_arg->rr_policy); } @@ -686,7 +686,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, if (!replace_old_rr) { /* dispose of the new RR policy that won't be used after all */ GRPC_LB_POLICY_UNREF(exec_ctx, new_rr_policy, "rr_handover_no_replace"); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Keeping old RR policy (%p) despite new serverlist: new RR " "policy was in %s connectivity state.", @@ -696,7 +696,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, return; } - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Created RR policy (%p) to replace old RR (%p)", (void *)new_rr_policy, (void *)glb_policy->rr_policy); } @@ -741,7 +741,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, pp->wrapped_on_complete_arg.rr_policy = glb_policy->rr_policy; pp->wrapped_on_complete_arg.client_stats = grpc_grpclb_client_stats_ref(glb_policy->client_stats); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Pending pick about to PICK from 0x%" PRIxPTR "", (intptr_t)glb_policy->rr_policy); } @@ -755,7 +755,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_policy->pending_pings = pping->next; GRPC_LB_POLICY_REF(glb_policy->rr_policy, "rr_handover_pending_ping"); pping->wrapped_notify_arg.rr_policy = glb_policy->rr_policy; - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Pending ping about to PING from 0x%" PRIxPTR "", (intptr_t)glb_policy->rr_policy); } @@ -908,7 +908,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, GPR_ASSERT(uri->path[0] != '\0'); glb_policy->server_name = gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.", glb_policy->server_name); } @@ -1093,7 +1093,7 @@ static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, bool pick_done; if (glb_policy->rr_policy != NULL) { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "grpclb %p about to PICK from RR %p", (void *)glb_policy, (void *)glb_policy->rr_policy); } @@ -1116,7 +1116,7 @@ static int glb_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pick_done = pick_from_internal_rr_locked(exec_ctx, glb_policy->rr_policy, pick_args, target, wc_arg); } else { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_DEBUG, "No RR policy in grpclb instance %p. Adding to grpclb's pending " "picks", @@ -1347,7 +1347,7 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, lb_call_init_locked(exec_ctx, glb_policy); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Query for backends (grpclb: %p, lb_call: %p)", (void *)glb_policy, (void *)glb_policy->lb_call); } @@ -1453,7 +1453,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, gpr_time_max(gpr_time_from_seconds(1, GPR_TIMESPAN), grpc_grpclb_duration_to_timespec( &response->client_stats_report_interval)); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "received initial LB response message; " "client load reporting interval = %" PRId64 ".%09d sec", @@ -1466,7 +1466,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, glb_policy->client_load_report_timer_pending = true; GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "client_load_report"); schedule_next_client_load_report(exec_ctx, glb_policy); - } else if (grpc_lb_glb_trace) { + } else if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "received initial LB response message; " "client load reporting NOT enabled"); @@ -1478,7 +1478,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_grpclb_response_parse_serverlist(response_slice); if (serverlist != NULL) { GPR_ASSERT(glb_policy->lb_call != NULL); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Serverlist with %lu servers received", (unsigned long)serverlist->num_servers); for (size_t i = 0; i < serverlist->num_servers; ++i) { @@ -1495,7 +1495,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (serverlist->num_servers > 0) { if (grpc_grpclb_serverlist_equals(glb_policy->serverlist, serverlist)) { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Incoming server list identical to current, ignoring."); } @@ -1513,7 +1513,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, rr_handover_locked(exec_ctx, glb_policy); } } else { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Received empty server list. Picks will stay pending until " "a response with > 0 servers is received"); @@ -1555,7 +1555,7 @@ static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, glb_lb_policy *glb_policy = arg; if (!glb_policy->shutting_down) { - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_INFO, "Restaring call to LB server (grpclb %p)", (void *)glb_policy); } @@ -1572,7 +1572,7 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, GPR_ASSERT(glb_policy->lb_call != NULL); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { char *status_details = grpc_slice_to_c_string(glb_policy->lb_call_status_details); gpr_log(GPR_DEBUG, @@ -1591,7 +1591,7 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); gpr_timespec next_try = gpr_backoff_step(&glb_policy->lb_call_backoff_state, now); - if (grpc_lb_glb_trace) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { gpr_log(GPR_DEBUG, "Connection to LB server lost (grpclb: %p)...", (void *)glb_policy); gpr_timespec timeout = gpr_time_sub(next_try, now); diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 4c17f9c0828..075b7caf5c7 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -74,7 +74,7 @@ typedef struct round_robin_lb_policy round_robin_lb_policy; -int grpc_lb_round_robin_trace = 0; +grpc_tracer_flag grpc_lb_round_robin_trace; /** List of entities waiting for a pick. * @@ -198,7 +198,7 @@ static void advance_last_picked_locked(round_robin_lb_policy *p) { GPR_ASSERT(p->ready_list_last_pick == &p->ready_list); } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[READYLIST, RR: %p] ADVANCED LAST PICK. NOW AT NODE %p (SC %p, " "CSC %p)", @@ -228,7 +228,7 @@ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p, p->ready_list.prev->next = new_elem; p->ready_list.prev = new_elem; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (Conn. SC %p)", (void *)new_elem, (void *)sd->subchannel); } @@ -256,7 +256,7 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p, node->next->prev = node->prev; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", (void *)node, (void *)node->subchannel); } @@ -276,7 +276,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { round_robin_lb_policy *p = (round_robin_lb_policy *)pol; ready_list *elem; - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "Destroying Round Robin policy at %p", (void *)pol); } @@ -312,7 +312,7 @@ static void rr_shutdown_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { pending_pick *pp; size_t i; - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "Shutting down Round Robin policy at %p", (void *)pol); } @@ -421,7 +421,7 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pending_pick *pp; ready_list *selected; - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_INFO, "Round Robin %p trying to pick", (void *)pol); } @@ -434,7 +434,7 @@ static int rr_pick_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, if (user_data != NULL) { *user_data = selected->user_data; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", (void *)*target, (void *)selected); @@ -566,7 +566,7 @@ static void rr_connectivity_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, if (pp->user_data != NULL) { *pp->user_data = selected->user_data; } - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", (void *)selected->subchannel, (void *)selected); @@ -724,7 +724,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sc_args.args = new_args; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { char *address_uri = grpc_sockaddr_to_uri(&addresses->addresses[i].address); gpr_log(GPR_DEBUG, "Created subchannel %p for address uri %s", @@ -768,7 +768,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, grpc_connectivity_state_init(&p->state_tracker, GRPC_CHANNEL_IDLE, "round_robin"); - if (grpc_lb_round_robin_trace) { + if (GRPC_TRACER_ON(grpc_lb_round_robin_trace)) { gpr_log(GPR_DEBUG, "Created RR policy at %p with %lu subchannels", (void *)p, (unsigned long)p->num_subchannels); } diff --git a/src/core/ext/filters/http/http_filters_plugin.c b/src/core/ext/filters/http/http_filters_plugin.c index 195a1a8119b..856a7dbd91e 100644 --- a/src/core/ext/filters/http/http_filters_plugin.c +++ b/src/core/ext/filters/http/http_filters_plugin.c @@ -37,6 +37,7 @@ #include "src/core/ext/filters/http/message_compress/message_compress_filter.h" #include "src/core/ext/filters/http/server/http_server_filter.h" #include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/transport/transport_impl.h" diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 1da8cf69cbf..5a54a6ed152 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -47,6 +47,7 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" +#include "src/core/lib/surface/call.h" #include "src/core/lib/transport/static_metadata.h" #define INITIAL_METADATA_UNSEEN 0 @@ -197,7 +198,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, did_compress = grpc_msg_compress(exec_ctx, calld->compression_algorithm, &calld->slices, &tmp); if (did_compress) { - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name; const size_t before_size = calld->slices.length; const size_t after_size = tmp.length; @@ -211,7 +212,7 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_swap(&calld->slices, &tmp); calld->send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name; GPR_ASSERT(grpc_compression_algorithm_name(calld->compression_algorithm, &algo_name)); diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.h b/src/core/ext/filters/http/message_compress/message_compress_filter.h index 75bfa17fba3..135da4da620 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.h +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.h @@ -38,8 +38,6 @@ #include "src/core/lib/channel/channel_stack.h" -extern int grpc_compression_trace; - /** Compression filter for outgoing data. * * See for the available compression settings. diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index d6b79bd4923..c5c0ec316d8 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -89,8 +89,8 @@ static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; #define MAX_CLIENT_STREAM_ID 0x7fffffffu -int grpc_http_trace = 0; -int grpc_flowctl_trace = 0; +grpc_tracer_flag grpc_http_trace; +grpc_tracer_flag grpc_flowctl_trace; static const grpc_transport_vtable vtable; @@ -1095,7 +1095,7 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, return; } closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { const char *errstr = grpc_error_string(error); gpr_log(GPR_DEBUG, "complete_closure_step: %p refs=%d flags=0x%04x desc=%s err=%s", @@ -1240,7 +1240,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, grpc_transport_stream_op_batch_payload *op_payload = op->payload; grpc_chttp2_transport *t = s->t; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *str = grpc_transport_stream_op_batch_string(op); gpr_log(GPR_DEBUG, "perform_stream_op_locked: %s; on_complete = %p", str, op->on_complete); @@ -1483,7 +1483,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *str = grpc_transport_stream_op_batch_string(op); gpr_log(GPR_DEBUG, "perform_stream_op[s=%p/%d]: %s", s, s->id, str); gpr_free(str); @@ -2145,7 +2145,7 @@ static void update_bdp(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (delta == 0 || (bdp != 0 && delta > -1024 && delta < 1024)) { return; } - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "%s: update initial window size to %d", t->peer_string, (int)bdp); } @@ -2305,7 +2305,7 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = tp; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "%s: Start BDP ping", t->peer_string); } /* Reset the keepalive ping timer */ @@ -2318,7 +2318,7 @@ static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = tp; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "%s: Complete BDP ping", t->peer_string); } grpc_bdp_estimator_complete_ping(&t->bdp_estimator); @@ -2779,7 +2779,7 @@ static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_chttp2_stream_map_size(&t->stream_map) == 0) { /* Channel with no active streams: send a goaway to try and make it * disconnect cleanly */ - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - send goaway to free memory", t->peer_string); } @@ -2787,7 +2787,7 @@ static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Buffers full"), GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_ENHANCE_YOUR_CALM)); - } else if (error == GRPC_ERROR_NONE && grpc_resource_quota_trace) { + } else if (error == GRPC_ERROR_NONE && GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR " streams", @@ -2808,7 +2808,7 @@ static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, t->destructive_reclaimer_registered = false; if (error == GRPC_ERROR_NONE && n > 0) { grpc_chttp2_stream *s = grpc_chttp2_stream_map_rand(&t->stream_map); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - abandon stream id %d", t->peer_string, s->id); } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index c372174f2d5..83b17d1936f 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -34,11 +34,12 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/transport/transport.h" -extern int grpc_http_trace; -extern int grpc_flowctl_trace; +extern grpc_tracer_flag grpc_http_trace; +extern grpc_tracer_flag grpc_flowctl_trace; grpc_transport *grpc_create_chttp2_transport( grpc_exec_ctx *exec_ctx, const grpc_channel_args *channel_args, diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index e3cd70d3f3b..dbaafb5929f 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -218,18 +218,18 @@ grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, void *p, parser->incoming_settings[id] != parser->value) { t->initial_window_update += (int64_t)parser->value - parser->incoming_settings[id]; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "adding %d for initial_window change", (int)t->initial_window_update); } } parser->incoming_settings[id] = parser->value; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "CHTTP2:%s:%s: got setting %s = %d", t->is_client ? "CLI" : "SVR", t->peer_string, sp->name, parser->value); } - } else if (grpc_http_trace) { + } else if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_ERROR, "CHTTP2: Ignoring unknown setting %d (value %d)", parser->id, parser->value); } diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 1846a85fc65..bb98bc4a79f 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -50,8 +50,6 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/http2_errors.h" -extern int grpc_http_trace; - typedef enum { NOT_BINARY, BINARY_BEGIN, @@ -666,7 +664,7 @@ static const uint8_t inverse_base64[256] = { /* emission helpers */ static grpc_error *on_hdr(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, grpc_mdelem md, int add_to_table) { - if (grpc_http_trace && !GRPC_MDELEM_IS_INTERNED(md)) { + if (GRPC_TRACER_ON(grpc_http_trace) && !GRPC_MDELEM_IS_INTERNED(md)) { char *k = grpc_slice_to_c_string(GRPC_MDKEY(md)); char *v = grpc_slice_to_c_string(GRPC_MDVALUE(md)); gpr_log( @@ -1052,7 +1050,7 @@ static grpc_error *parse_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, static grpc_error *finish_max_tbl_size(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index); } grpc_error *err = diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 0aaa4aebe5f..bb5ce60872e 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -629,13 +629,13 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, #define GRPC_CHTTP2_CLIENT_CONNECT_STRLEN \ (sizeof(GRPC_CHTTP2_CLIENT_CONNECT_STRING) - 1) -extern int grpc_http_trace; -extern int grpc_flowctl_trace; +extern grpc_tracer_flag grpc_http_trace; +extern grpc_tracer_flag grpc_flowctl_trace; -#define GRPC_CHTTP2_IF_TRACING(stmt) \ - if (!(grpc_http_trace)) \ - ; \ - else \ +#define GRPC_CHTTP2_IF_TRACING(stmt) \ + if (!(GRPC_TRACER_ON(grpc_http_trace))) \ + ; \ + else \ stmt typedef enum { @@ -648,7 +648,7 @@ typedef enum { dst_var, src_context, src_var) \ do { \ assert(id1 == id2); \ - if (grpc_flowctl_trace) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace( \ __FILE__, __LINE__, phase, GRPC_CHTTP2_FLOWCTL_MOVE, #dst_context, \ #dst_var, #src_context, #src_var, transport->is_client, id1, \ @@ -671,7 +671,7 @@ typedef enum { #define GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ - if (grpc_flowctl_trace) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ GRPC_CHTTP2_FLOWCTL_CREDIT, #dst_context, \ #dst_var, NULL, #amount, transport->is_client, \ @@ -729,7 +729,7 @@ typedef enum { #define GRPC_CHTTP2_FLOW_DEBIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ - if (grpc_flowctl_trace) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ GRPC_CHTTP2_FLOWCTL_DEBIT, #dst_context, \ #dst_var, NULL, #amount, transport->is_client, \ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 638b137316f..cee3c3f4329 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -324,7 +324,7 @@ static grpc_error *init_frame_parser(grpc_exec_ctx *exec_ctx, case GRPC_CHTTP2_FRAME_GOAWAY: return init_goaway_parser(exec_ctx, t); default: - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_ERROR, "Unknown frame type %02x", t->incoming_frame_type); } return init_skip_frame_parser(exec_ctx, t, 0); @@ -494,7 +494,7 @@ static void on_initial_header(grpc_exec_ctx *exec_ctx, void *tp, GPR_ASSERT(s != NULL); - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *key = grpc_slice_to_c_string(GRPC_MDKEY(md)); char *value = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX | GPR_DUMP_ASCII); @@ -574,7 +574,7 @@ static void on_trailing_header(grpc_exec_ctx *exec_ctx, void *tp, GPR_ASSERT(s != NULL); - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { char *key = grpc_slice_to_c_string(GRPC_MDKEY(md)); char *value = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX | GPR_DUMP_ASCII); @@ -807,7 +807,7 @@ static grpc_error *parse_frame_slice(grpc_exec_ctx *exec_ctx, if (err == GRPC_ERROR_NONE) { return err; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { const char *msg = grpc_error_string(err); gpr_log(GPR_ERROR, "%s", msg); } diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 069780ae5af..54ff0aadad8 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -74,7 +74,8 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, } if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_INFLIGHT])) { /* ping already in-flight: wait */ - if (grpc_http_trace || grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: already pinging", t->peer_string); } return; @@ -82,7 +83,8 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, if (t->ping_state.pings_before_data_required == 0 && t->ping_policy.max_pings_without_data != 0) { /* need to send something of substance before sending a ping again */ - if (grpc_http_trace || grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: too many recent pings: %d/%d", t->peer_string, t->ping_state.pings_before_data_required, t->ping_policy.max_pings_without_data); @@ -96,7 +98,8 @@ static void maybe_initiate_ping(grpc_exec_ctx *exec_ctx, (int)t->ping_policy.min_time_between_pings.tv_nsec);*/ if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < 0) { /* not enough elapsed time between successive pings */ - if (grpc_http_trace || grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_http_trace) || + GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "Ping delayed [%p]: not enough time elapsed since last ping", t->peer_string); diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 94382980eb3..107d5997fdf 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -38,7 +38,7 @@ #include #include -int grpc_trace_channel = 0; +grpc_tracer_flag grpc_trace_channel; /* Memory layouts. diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index fdbcbdb0185..c26d61b2ef4 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -307,10 +307,10 @@ void grpc_call_element_signal_error(grpc_exec_ctx *exec_ctx, grpc_call_element *cur_elem, grpc_error *error); -extern int grpc_trace_channel; +extern grpc_tracer_flag grpc_trace_channel; #define GRPC_CALL_LOG_OP(sev, elem, op) \ - if (grpc_trace_channel) grpc_call_log_op(sev, elem, op) + if (GRPC_TRACER_ON(grpc_trace_channel)) grpc_call_log_op(sev, elem, op) #ifdef __cplusplus } diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index 88c02edb70e..332635f58fb 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -38,7 +38,7 @@ #include #include -int grpc_trace_channel_stack_builder = 0; +grpc_tracer_flag grpc_trace_channel_stack_builder; typedef struct filter_node { struct filter_node *next; diff --git a/src/core/lib/channel/channel_stack_builder.h b/src/core/lib/channel/channel_stack_builder.h index c78111b00d0..8cb36eb117e 100644 --- a/src/core/lib/channel/channel_stack_builder.h +++ b/src/core/lib/channel/channel_stack_builder.h @@ -165,7 +165,7 @@ grpc_error *grpc_channel_stack_builder_finish( void grpc_channel_stack_builder_destroy(grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder); -extern int grpc_trace_channel_stack_builder; +extern grpc_tracer_flag grpc_trace_channel_stack_builder; #ifdef __cplusplus } diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c index c56046785b3..54653868b83 100644 --- a/src/core/lib/debug/trace.c +++ b/src/core/lib/debug/trace.c @@ -42,17 +42,23 @@ typedef struct tracer { const char *name; - int *flag; + grpc_tracer_flag *flag; struct tracer *next; } tracer; static tracer *tracers; -void grpc_register_tracer(const char *name, int *flag) { +#ifdef GRPC_THREADSAFE_TRACER +#define TRACER_SET(flag, on) gpr_atm_no_barrier_store(&(flag).value, (on)) +#else +#define TRACER_SET(flag, on) (flag).value = (on) +#endif + +void grpc_register_tracer(const char *name, grpc_tracer_flag *flag) { tracer *t = gpr_malloc(sizeof(*t)); t->name = name; t->flag = flag; t->next = tracers; - *flag = 0; + TRACER_SET(*flag, false); tracers = t; } @@ -121,13 +127,13 @@ int grpc_tracer_set_enabled(const char *name, int enabled) { tracer *t; if (0 == strcmp(name, "all")) { for (t = tracers; t; t = t->next) { - *t->flag = enabled; + TRACER_SET(*t->flag, enabled); } } else { int found = 0; for (t = tracers; t; t = t->next) { if (0 == strcmp(name, t->name)) { - *t->flag = enabled; + TRACER_SET(*t->flag, enabled); found = 1; } } diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index 7afc38db7e1..bf3bfee306f 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -34,9 +34,33 @@ #ifndef GRPC_CORE_LIB_DEBUG_TRACE_H #define GRPC_CORE_LIB_DEBUG_TRACE_H +#include #include +#include -void grpc_register_tracer(const char *name, int *flag); +#if defined(__has_feature) && __has_feature(thread_sanitizer) +#define GRPC_THREADSAFE_TRACER +#endif + +typedef struct { +#ifdef GRPC_THREADSAFE_TRACER + gpr_atm value; +#else + bool value; +#endif +} grpc_tracer_flag; + +#ifdef GRPC_THREADSAFE_TRACER +#define GRPC_TRACER_ON(flag) (gpr_atm_no_barrier_load(&(flag).value) != 0) +#define GRPC_TRACER_INITIALIZER(on) \ + { (gpr_atm)(on) } +#else +#define GRPC_TRACER_ON(flag) ((flag).value) +#define GRPC_TRACER_INITIALIZER(on) \ + { (on) } +#endif + +void grpc_register_tracer(const char *name, grpc_tracer_flag *flag); void grpc_tracer_init(const char *env_var_name); void grpc_tracer_shutdown(void); diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index aac506b800d..b8883f5953e 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -40,7 +40,7 @@ #include #include -int grpc_http1_trace = 0; +grpc_tracer_flag grpc_http1_trace; static char *buf2str(void *buffer, size_t length) { char *out = gpr_malloc(length + 1); @@ -308,7 +308,7 @@ static grpc_error *addbyte(grpc_http_parser *parser, uint8_t byte, case GRPC_HTTP_FIRST_LINE: case GRPC_HTTP_HEADERS: if (parser->cur_line_length >= GRPC_HTTP_PARSER_MAX_HEADER_LENGTH) { - if (grpc_http1_trace) + if (GRPC_TRACER_ON(grpc_http1_trace)) gpr_log(GPR_ERROR, "HTTP header max line length (%d) exceeded", GRPC_HTTP_PARSER_MAX_HEADER_LENGTH); return GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/core/lib/http/parser.h b/src/core/lib/http/parser.h index a68011dd439..a155fecf118 100644 --- a/src/core/lib/http/parser.h +++ b/src/core/lib/http/parser.h @@ -36,6 +36,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/error.h" /* Maximum length of a header string of the form 'Key: Value\r\n' */ @@ -121,6 +122,6 @@ grpc_error *grpc_http_parser_eof(grpc_http_parser *parser); void grpc_http_request_destroy(grpc_http_request *request); void grpc_http_response_destroy(grpc_http_response *response); -extern int grpc_http1_trace; +extern grpc_tracer_flag grpc_http1_trace; #endif /* GRPC_CORE_LIB_HTTP_PARSER_H */ diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index 05cdbdad2b7..fa7d576133f 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -42,13 +42,13 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" -int grpc_combiner_trace = 0; +grpc_tracer_flag grpc_combiner_trace; -#define GRPC_COMBINER_TRACE(fn) \ - do { \ - if (grpc_combiner_trace) { \ - fn; \ - } \ +#define GRPC_COMBINER_TRACE(fn) \ + do { \ + if (GRPC_TRACER_ON(grpc_combiner_trace)) { \ + fn; \ + } \ } while (0) #define STATE_UNORPHANED 1 diff --git a/src/core/lib/iomgr/combiner.h b/src/core/lib/iomgr/combiner.h index 75dcb0b70a4..6ab7a2b26b2 100644 --- a/src/core/lib/iomgr/combiner.h +++ b/src/core/lib/iomgr/combiner.h @@ -37,6 +37,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/mpscq.h" @@ -78,6 +79,6 @@ grpc_closure_scheduler *grpc_combiner_finally_scheduler(grpc_combiner *lock, bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx); -extern int grpc_combiner_trace; +extern grpc_tracer_flag grpc_combiner_trace; #endif /* GRPC_CORE_LIB_IOMGR_COMBINER_H */ diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 445f7aa422d..37804d86009 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -66,6 +66,9 @@ void grpc_iomgr_init(void) { g_root_object.name = "root"; grpc_network_status_init(); grpc_iomgr_platform_init(); +} + +void grpc_iomgr_start(void) { grpc_timer_manager_init(); } diff --git a/src/core/lib/iomgr/iomgr.h b/src/core/lib/iomgr/iomgr.h index 245a1e08aa2..6e2e0236154 100644 --- a/src/core/lib/iomgr/iomgr.h +++ b/src/core/lib/iomgr/iomgr.h @@ -40,6 +40,9 @@ /** Initializes the iomgr. */ void grpc_iomgr_init(void); +/** Starts any background threads for iomgr. */ +void grpc_iomgr_start(void); + /** Signals the intention to shutdown the iomgr. Expects to be able to flush * exec_ctx. */ void grpc_iomgr_shutdown(grpc_exec_ctx *exec_ctx); diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index c3ee8786517..52ccfc0aad1 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -44,7 +44,7 @@ #include "src/core/lib/iomgr/combiner.h" -int grpc_resource_quota_trace = 0; +grpc_tracer_flag grpc_resource_quota_trace; #define MEMORY_USAGE_ESTIMATION_MAX 65536 @@ -307,13 +307,13 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, resource_user->free_pool = 0; resource_quota->free_pool -= amt; rq_update_estimate(resource_quota); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64 " bytes; rq_free_pool -> %" PRId64, resource_quota->name, resource_user->name, amt, resource_quota->free_pool); } - } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) { + } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && resource_user->free_pool >= 0) { gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request", resource_quota->name, resource_user->name); } @@ -342,7 +342,7 @@ static bool rq_reclaim_from_per_user_free_pool( resource_user->free_pool = 0; resource_quota->free_pool += amt; rq_update_estimate(resource_quota); - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 " bytes; rq_free_pool -> %" PRId64, resource_quota->name, resource_user->name, amt, @@ -365,7 +365,7 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx, : GRPC_RULIST_RECLAIMER_BENIGN; grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list); if (resource_user == NULL) return false; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation", resource_quota->name, resource_user->name, destructive ? "destructive" : "benign"); @@ -786,7 +786,7 @@ void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&resource_user->mu); ru_ref_by(resource_user, (gpr_atm)size); resource_user->free_pool -= (int64_t)size; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, resource_user->free_pool); @@ -810,7 +810,7 @@ void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&resource_user->mu); bool was_zero_or_negative = resource_user->free_pool <= 0; resource_user->free_pool += (int64_t)size; - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; free_pool -> %" PRId64, resource_user->resource_quota->name, resource_user->name, size, resource_user->free_pool); @@ -839,7 +839,7 @@ void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx, grpc_resource_user *resource_user) { - if (grpc_resource_quota_trace) { + if (GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete", resource_user->resource_quota->name, resource_user->name); } diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index 6f99be0d512..51122dad011 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -36,6 +36,7 @@ #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" /** \file Tracks resource usage against a pool. @@ -75,7 +76,7 @@ maintain lists of users (which users arrange to leave before they are destroyed) */ -extern int grpc_resource_quota_trace; +extern grpc_tracer_flag grpc_resource_quota_trace; grpc_resource_quota *grpc_resource_quota_ref_internal( grpc_resource_quota *resource_quota); diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index a2692707d9e..ed3fd94a98d 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -58,7 +58,7 @@ #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/support/string.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; typedef struct { gpr_mu mu; @@ -114,7 +114,7 @@ done: static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { int done; async_connect *ac = acp; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", ac->addr_str, str); @@ -152,7 +152,7 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { GRPC_ERROR_REF(error); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_writable: error=%s", ac->addr_str, str); @@ -330,7 +330,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, grpc_schedule_on_exec_ctx); ac->channel_args = grpc_channel_args_copy(channel_args); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", ac->addr_str); } diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 5f4b38de2b9..233dea9bece 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -74,7 +74,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type; typedef size_t msg_iovlen_type; #endif -int grpc_tcp_trace = 0; +grpc_tracer_flag grpc_tcp_trace; typedef struct { grpc_endpoint base; @@ -221,7 +221,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp, grpc_error *error) { grpc_closure *cb = tcp->read_cb; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -468,14 +468,14 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */, } if (!tcp_flush(tcp, &error)) { - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "write: delayed"); } grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { cb = tcp->write_cb; tcp->write_cb = NULL; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write: %s", str); } @@ -490,7 +490,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_tcp *tcp = (grpc_tcp *)ep; grpc_error *error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; for (i = 0; i < buf->count; i++) { @@ -521,12 +521,12 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, if (!tcp_flush(tcp, &error)) { TCP_REF(tcp, "write"); tcp->write_cb = cb; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "write: delayed"); } grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure); } else { - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write: %s", str); } diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index 1ad5788331f..4ad60c116eb 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -44,10 +44,11 @@ otherwise specified. */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/ev_posix.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; /* Create a tcp endpoint given a file desciptor and a read slice size. Takes ownership of fd. */ diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index e66ffc9b1c2..08997b5e2bb 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -257,7 +257,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { addr_str = grpc_sockaddr_to_uri(&addr); gpr_asprintf(&name, "tcp-server-connection:%s", addr_str); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "SERVER_CONNECT: incoming connection: %s", addr_str); } diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index d8e60684312..ec0bb169cb3 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -56,8 +56,8 @@ #define MIN_QUEUE_WINDOW_DURATION 0.01 #define MAX_QUEUE_WINDOW_DURATION 1 -int grpc_timer_trace = 0; -int grpc_timer_check_trace = 0; +grpc_tracer_flag grpc_timer_trace; +grpc_tracer_flag grpc_timer_check_trace; typedef struct { gpr_mu mu; @@ -232,14 +232,13 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, GPR_ASSERT(deadline.clock_type == g_clock_type); GPR_ASSERT(now.clock_type == g_clock_type); timer->closure = closure; - timer->deadline = timespec_to_atm_round_up(deadline); + gpr_atm deadline_atm = timer->deadline = timespec_to_atm_round_up(deadline); - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, "TIMER %p: SET %" PRId64 ".%09d [%" PRIdPTR "] now %" PRId64 ".%09d [%" PRIdPTR "] call %p[%p]", - timer, deadline.tv_sec, deadline.tv_nsec, timer->deadline, - now.tv_sec, now.tv_nsec, timespec_to_atm_round_down(now), closure, - closure->cb); + timer, deadline.tv_sec, deadline.tv_nsec, deadline_atm, now.tv_sec, + now.tv_nsec, timespec_to_atm_round_down(now), closure, closure->cb); } if (!g_shared_mutables.initialized) { @@ -262,13 +261,13 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, grpc_time_averaged_stats_add_sample(&shard->stats, ts_to_dbl(gpr_time_sub(deadline, now))); - if (timer->deadline < shard->queue_deadline_cap) { + if (deadline_atm < shard->queue_deadline_cap) { is_first_timer = grpc_timer_heap_add(&shard->heap, timer); } else { timer->heap_index = INVALID_HEAP_INDEX; list_join(&shard->list, timer); } - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, " .. add to shard %d with queue_deadline_cap=%" PRIdPTR " => is_first_timer=%s", (int)(shard - g_shards), shard->queue_deadline_cap, @@ -289,16 +288,16 @@ void grpc_timer_init(grpc_exec_ctx *exec_ctx, grpc_timer *timer, grpc_timer_check. */ if (is_first_timer) { gpr_mu_lock(&g_shared_mutables.mu); - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, " .. old shard min_deadline=%" PRIdPTR, shard->min_deadline); } - if (timer->deadline < shard->min_deadline) { + if (deadline_atm < shard->min_deadline) { gpr_atm old_min_deadline = g_shard_queue[0]->min_deadline; - shard->min_deadline = timer->deadline; + shard->min_deadline = deadline_atm; note_deadline_change(shard); - if (shard->shard_queue_index == 0 && timer->deadline < old_min_deadline) { - gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, timer->deadline); + if (shard->shard_queue_index == 0 && deadline_atm < old_min_deadline) { + gpr_atm_no_barrier_store(&g_shared_mutables.min_timer, deadline_atm); grpc_kick_poller(); } } @@ -319,7 +318,7 @@ void grpc_timer_cancel(grpc_exec_ctx *exec_ctx, grpc_timer *timer) { shard_type *shard = &g_shards[GPR_HASH_POINTER(timer, NUM_SHARDS)]; gpr_mu_lock(&shard->mu); - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, "TIMER %p: CANCEL pending=%s", timer, timer->pending ? "true" : "false"); } @@ -355,7 +354,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) { saturating_add(GPR_MAX(now, shard->queue_deadline_cap), (gpr_atm)(deadline_delta * 1000.0)); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. shard[%d]->queue_deadline_cap --> %" PRIdPTR, (int)(shard - g_shards), shard->queue_deadline_cap); } @@ -363,7 +362,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) { next = timer->next; if (timer->deadline < shard->queue_deadline_cap) { - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. add timer with deadline %" PRIdPTR " to heap", timer->deadline); } @@ -380,7 +379,7 @@ static int refill_queue(shard_type *shard, gpr_atm now) { static grpc_timer *pop_one(shard_type *shard, gpr_atm now) { grpc_timer *timer; for (;;) { - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. shard[%d]: heap_empty=%s", (int)(shard - g_shards), grpc_timer_heap_is_empty(&shard->heap) ? "true" : "false"); @@ -390,13 +389,13 @@ static grpc_timer *pop_one(shard_type *shard, gpr_atm now) { if (!refill_queue(shard, now)) return NULL; } timer = grpc_timer_heap_top(&shard->heap); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. check top timer deadline=%" PRIdPTR " now=%" PRIdPTR, timer->deadline, now); } if (timer->deadline > now) return NULL; - if (grpc_timer_trace) { + if (GRPC_TRACER_ON(grpc_timer_trace)) { gpr_log(GPR_DEBUG, "TIMER %p: FIRE %" PRIdPTR "ms late", timer, now - timer->deadline); } @@ -436,7 +435,7 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now, if (gpr_spinlock_trylock(&g_shared_mutables.checker_mu)) { gpr_mu_lock(&g_shared_mutables.mu); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. shard[%d]->min_deadline = %" PRIdPTR, (int)(g_shard_queue[0] - g_shards), g_shard_queue[0]->min_deadline); @@ -452,7 +451,7 @@ static int run_some_expired_timers(grpc_exec_ctx *exec_ctx, gpr_atm now, n += pop_timers(exec_ctx, g_shard_queue[0], now, &new_min_deadline, error); - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, " .. popped --> %" PRIdPTR ", shard[%d]->min_deadline %" PRIdPTR " --> %" PRIdPTR ", now=%" PRIdPTR, @@ -509,7 +508,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, *next = atm_to_timespec(GPR_MIN(timespec_to_atm_round_up(*next), min_timer)); } - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { gpr_log(GPR_DEBUG, "TIMER CHECK SKIP: now_atm=%" PRIdPTR " min_timer=%" PRIdPTR, now_atm, min_timer); @@ -523,7 +522,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, : GRPC_ERROR_CREATE_FROM_STATIC_STRING("Shutting down timer system"); // tracing - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { char *next_str; if (next == NULL) { next_str = gpr_strdup("NULL"); @@ -549,7 +548,7 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, *next = atm_to_timespec(next_atm); } // tracing - if (grpc_timer_check_trace) { + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { char *next_str; if (next == NULL) { next_str = gpr_strdup("NULL"); diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 00e868de016..1d83341299e 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -45,11 +45,11 @@ typedef struct completed_thread { } completed_thread; static gpr_mu g_mu; +static bool g_threaded; static gpr_cv g_cv_wait; static gpr_cv g_cv_shutdown; static int g_thread_count; static int g_waiter_count; -static bool g_shutdown; static completed_thread *g_completed_threads; static bool g_kicked; @@ -83,6 +83,14 @@ static void start_timer_thread_and_unlock(void) { gpr_thd_new(&thd, timer_thread, NULL, &opt); } +void grpc_timer_manager_tick() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + grpc_timer_check(&exec_ctx, now, &next); + grpc_exec_ctx_finish(&exec_ctx); +} + static void timer_thread(void *unused) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); @@ -93,7 +101,7 @@ static void timer_thread(void *unused) { gpr_mu_lock(&g_mu); --g_waiter_count; bool start_thread = g_waiter_count == 0; - if (start_thread && !g_shutdown) { + if (start_thread && g_threaded) { start_timer_thread_and_unlock(); } else { gpr_mu_unlock(&g_mu); @@ -105,7 +113,7 @@ static void timer_thread(void *unused) { gpr_mu_unlock(&g_mu); } else { gpr_mu_lock(&g_mu); - if (g_shutdown) break; + if (!g_threaded) break; if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) { if (g_kicked) { grpc_timer_consume_kick(); @@ -130,34 +138,58 @@ static void timer_thread(void *unused) { gpr_log(GPR_DEBUG, "End timer thread"); } +static void start_threads(void) { + gpr_mu_lock(&g_mu); + if (!g_threaded) { + g_threaded = true; + start_timer_thread_and_unlock(); + } else { + g_threaded = false; + gpr_mu_unlock(&g_mu); + } +} + void grpc_timer_manager_init(void) { gpr_mu_init(&g_mu); gpr_cv_init(&g_cv_wait); gpr_cv_init(&g_cv_shutdown); + g_threaded = false; g_thread_count = 0; g_waiter_count = 0; - g_shutdown = false; g_completed_threads = NULL; - gpr_mu_lock(&g_mu); - start_timer_thread_and_unlock(); + start_threads(); } -void grpc_timer_manager_shutdown(void) { +static void stop_threads(void) { gpr_mu_lock(&g_mu); - g_shutdown = true; - gpr_cv_broadcast(&g_cv_wait); - while (g_thread_count > 0) { - gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); - gc_completed_threads(); + if (g_threaded) { + g_threaded = false; + gpr_cv_broadcast(&g_cv_wait); + while (g_thread_count > 0) { + gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + gc_completed_threads(); + } } gpr_mu_unlock(&g_mu); +} + +void grpc_timer_manager_shutdown(void) { + stop_threads(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_cv_wait); gpr_cv_destroy(&g_cv_shutdown); } +void grpc_timer_manager_set_threading(bool threaded) { + if (threaded) { + start_threads(); + } else { + stop_threads(); + } +} + void grpc_kick_poller(void) { gpr_mu_lock(&g_mu); g_kicked = true; diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h index a24c9da328a..0b21262b1a2 100644 --- a/src/core/lib/iomgr/timer_manager.h +++ b/src/core/lib/iomgr/timer_manager.h @@ -34,10 +34,19 @@ #ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H #define GRPC_CORE_IOMGR_TIMER_MANAGER_H +#include + /* Timer Manager tries to keep one thread waiting for the next timeout at all times */ void grpc_timer_manager_init(void); void grpc_timer_manager_shutdown(void); +/* enable/disable threading - must be called after grpc_timer_manager_init and + * before grpc_timer_manager_shutdown */ +void grpc_timer_manager_set_threading(bool enabled); +/* explicitly perform one tick of the timer system - for when threading is + * disabled */ +void grpc_timer_manager_tick(void); + #endif diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c index 178ce89aa69..0e7c1afb024 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.c +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c @@ -171,7 +171,7 @@ static char *redact_private_key(const char *json_key) { grpc_call_credentials *grpc_service_account_jwt_access_credentials_create( const char *json_key, gpr_timespec token_lifetime, void *reserved) { - if (grpc_api_trace) { + if (GRPC_TRACER_ON(grpc_api_trace)) { char *clean_json = redact_private_key(json_key); gpr_log(GPR_INFO, "grpc_service_account_jwt_access_credentials_create(" diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c index ccfb3566c11..29235b6eb30 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c @@ -412,7 +412,7 @@ grpc_call_credentials *grpc_google_refresh_token_credentials_create( const char *json_refresh_token, void *reserved) { grpc_auth_refresh_token token = grpc_auth_refresh_token_create_from_string(json_refresh_token); - if (grpc_api_trace) { + if (GRPC_TRACER_ON(grpc_api_trace)) { char *loggable_token = create_loggable_refresh_token(&token); gpr_log(GPR_INFO, "grpc_refresh_token_credentials_create(json_refresh_token=%s, " diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index 1f0daf73253..dff05633ecb 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -253,7 +253,7 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_linked_mdelem *l; grpc_client_security_context *sec_ctx = NULL; - if (calld->security_context_set == 0 && !op->cancel_stream) { + if (!op->cancel_stream && calld->security_context_set == 0) { calld->security_context_set = 1; GPR_ASSERT(op->payload->context != NULL); if (op->payload->context[GRPC_CONTEXT_SECURITY].value == NULL) { diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 0d5c7432c64..c8dc50206ae 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -75,7 +75,7 @@ typedef struct { gpr_refcount ref; } secure_endpoint; -int grpc_trace_secure_endpoint = 0; +grpc_tracer_flag grpc_trace_secure_endpoint; static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; @@ -137,7 +137,7 @@ static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur, static void call_read_cb(grpc_exec_ctx *exec_ctx, secure_endpoint *ep, grpc_error *error) { - if (grpc_trace_secure_endpoint) { + if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) { size_t i; for (i = 0; i < ep->read_buffer->count; i++) { char *data = grpc_dump_slice(ep->read_buffer->slices[i], @@ -269,7 +269,7 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &ep->output_buffer); - if (grpc_trace_secure_endpoint) { + if (GRPC_TRACER_ON(grpc_trace_secure_endpoint)) { for (i = 0; i < slices->count; i++) { char *data = grpc_dump_slice(slices->slices[i], GPR_DUMP_HEX | GPR_DUMP_ASCII); diff --git a/src/core/lib/security/transport/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h index a61f40a4fa3..f1a5c8cb6de 100644 --- a/src/core/lib/security/transport/secure_endpoint.h +++ b/src/core/lib/security/transport/secure_endpoint.h @@ -39,7 +39,7 @@ struct tsi_frame_protector; -extern int grpc_trace_secure_endpoint; +extern grpc_tracer_flag grpc_trace_secure_endpoint; /* Takes ownership of protector and to_wrap, and refs leftover_slices. */ grpc_endpoint *grpc_secure_endpoint_create( diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 79e3e5ca9b8..3b20e042b47 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -33,4 +33,4 @@ #include "src/core/lib/surface/api_trace.h" -int grpc_api_trace = 0; +grpc_tracer_flag grpc_api_trace; diff --git a/src/core/lib/surface/api_trace.h b/src/core/lib/surface/api_trace.h index c60aaba5e97..d4fbc8d90d3 100644 --- a/src/core/lib/surface/api_trace.h +++ b/src/core/lib/surface/api_trace.h @@ -37,7 +37,7 @@ #include #include "src/core/lib/debug/trace.h" -extern int grpc_api_trace; +extern grpc_tracer_flag grpc_api_trace; /* Provide unwrapping macros because we're in C89 and variadic macros weren't introduced until C99... */ @@ -58,7 +58,7 @@ extern int grpc_api_trace; /* Due to the limitations of C89's preprocessor, the arity of the var-arg list 'nargs' must be specified. */ #define GRPC_API_TRACE(fmt, nargs, args) \ - if (grpc_api_trace) { \ + if (GRPC_TRACER_ON(grpc_api_trace)) { \ gpr_log(GPR_INFO, fmt GRPC_API_TRACE_UNWRAP##nargs args); \ } diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 75258065830..fbced008603 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -244,8 +244,8 @@ struct grpc_call { void *saved_receiving_stream_ready_bctlp; }; -int grpc_call_error_trace = 0; -int grpc_compression_trace = 0; +grpc_tracer_flag grpc_call_error_trace; +grpc_tracer_flag grpc_compression_trace; #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) @@ -702,7 +702,7 @@ static void get_final_status(grpc_call *call, for (i = 0; i < STATUS_SOURCE_COUNT; i++) { status[i] = unpack_received_status(gpr_atm_acq_load(&call->status[i])); } - if (grpc_call_error_trace) { + if (GRPC_TRACER_ON(grpc_call_error_trace)) { gpr_log(GPR_DEBUG, "get_final_status %s", call->is_client ? "CLI" : "SVR"); for (i = 0; i < STATUS_SOURCE_COUNT; i++) { if (status[i].is_set) { @@ -1259,7 +1259,7 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, } if (error != GRPC_ERROR_NONE) { - if (grpc_trace_operation_failures) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures)) { GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); } grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); @@ -1355,8 +1355,7 @@ static void validate_filtered_metadata(grpc_exec_ctx *exec_ctx, GPR_ASSERT(call->encodings_accepted_by_peer != 0); if (!GPR_BITGET(call->encodings_accepted_by_peer, call->incoming_compression_algorithm)) { - extern int grpc_compression_trace; - if (grpc_compression_trace) { + if (GRPC_TRACER_ON(grpc_compression_trace)) { char *algo_name = NULL; grpc_compression_algorithm_name(call->incoming_compression_algorithm, &algo_name); diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 7d4d0db28d9..256a5fa2feb 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -117,7 +117,8 @@ void grpc_call_context_set(grpc_call *call, grpc_context_index elem, void *grpc_call_context_get(grpc_call *call, grpc_context_index elem); #define GRPC_CALL_LOG_BATCH(sev, call, ops, nops, tag) \ - if (grpc_api_trace) grpc_call_log_batch(sev, call, ops, nops, tag) + if (GRPC_TRACER_ON(grpc_api_trace)) \ + grpc_call_log_batch(sev, call, ops, nops, tag) uint8_t grpc_call_is_client(grpc_call *call); @@ -126,7 +127,8 @@ uint8_t grpc_call_is_client(grpc_call *call); grpc_compression_algorithm grpc_call_compression_for_level( grpc_call *call, grpc_compression_level level); -extern int grpc_call_error_trace; +extern grpc_tracer_flag grpc_call_error_trace; +extern grpc_tracer_flag grpc_compression_trace; #ifdef __cplusplus } diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 048564e32f9..f31f0bc4c7c 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -50,9 +50,9 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/event_string.h" -int grpc_trace_operation_failures; +grpc_tracer_flag grpc_trace_operation_failures; #ifndef NDEBUG -int grpc_trace_pending_tags; +grpc_tracer_flag grpc_trace_pending_tags; #endif typedef struct { @@ -242,15 +242,16 @@ struct grpc_completion_queue { #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1)) #define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1) -int grpc_cq_pluck_trace; -int grpc_cq_event_timeout_trace; +grpc_tracer_flag grpc_cq_pluck_trace = GRPC_TRACER_INITIALIZER(true); +grpc_tracer_flag grpc_cq_event_timeout_trace = GRPC_TRACER_INITIALIZER(true); -#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \ - if (grpc_api_trace && \ - (grpc_cq_pluck_trace || (event)->type != GRPC_QUEUE_TIMEOUT)) { \ - char *_ev = grpc_event_string(event); \ - gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \ - gpr_free(_ev); \ +#define GRPC_SURFACE_TRACE_RETURNED_EVENT(cq, event) \ + if (GRPC_TRACER_ON(grpc_api_trace) && \ + (GRPC_TRACER_ON(grpc_cq_pluck_trace) || \ + (event)->type != GRPC_QUEUE_TIMEOUT)) { \ + char *_ev = grpc_event_string(event); \ + gpr_log(GPR_INFO, "RETURN_EVENT[%p]: %s", cq, _ev); \ + gpr_free(_ev); \ } static void on_pollset_shutdown_done(grpc_exec_ctx *exec_ctx, void *cc, @@ -375,14 +376,16 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, #endif GPR_TIMER_BEGIN("grpc_cq_end_op", 0); - if (grpc_api_trace || - (grpc_trace_operation_failures && error != GRPC_ERROR_NONE)) { + if (GRPC_TRACER_ON(grpc_api_trace) || + (GRPC_TRACER_ON(grpc_trace_operation_failures) && + error != GRPC_ERROR_NONE)) { const char *errmsg = grpc_error_string(error); GRPC_API_TRACE( "grpc_cq_end_op(exec_ctx=%p, cc=%p, tag=%p, error=%s, done=%p, " "done_arg=%p, storage=%p)", 7, (exec_ctx, cc, tag, errmsg, done, done_arg, storage)); - if (grpc_trace_operation_failures && error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_trace_operation_failures) && + error != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg); } } @@ -481,7 +484,7 @@ static bool cq_is_next_finished(grpc_exec_ctx *exec_ctx, void *arg) { #ifndef NDEBUG static void dump_pending_tags(grpc_completion_queue *cc) { - if (!grpc_trace_pending_tags) return; + if (!GRPC_TRACER_ON(grpc_trace_pending_tags)) return; gpr_strvec v; gpr_strvec_init(&v); @@ -677,7 +680,7 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, abort(); } - if (grpc_cq_pluck_trace) { + if (GRPC_TRACER_ON(grpc_cq_pluck_trace)) { GRPC_API_TRACE( "grpc_completion_queue_pluck(" "cc=%p, tag=%p, " diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index a932087939d..a750687b27d 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -37,15 +37,16 @@ /* Internal API for completion queues */ #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/pollset.h" /* These trace flags default to 1. The corresponding lines are only traced if grpc_api_trace is also truthy */ -extern int grpc_cq_pluck_trace; -extern int grpc_cq_event_timeout_trace; -extern int grpc_trace_operation_failures; +extern grpc_tracer_flag grpc_cq_pluck_trace; +extern grpc_tracer_flag grpc_cq_event_timeout_trace; +extern grpc_tracer_flag grpc_trace_operation_failures; #ifndef NDEBUG -extern int grpc_trace_pending_tags; +extern grpc_tracer_flag grpc_trace_pending_tags; #endif typedef struct grpc_cq_completion { diff --git a/src/core/lib/surface/init.c b/src/core/lib/surface/init.c index 4b381b19546..6163776152b 100644 --- a/src/core/lib/surface/init.c +++ b/src/core/lib/surface/init.c @@ -145,10 +145,8 @@ void grpc_init(void) { grpc_register_tracer("server_channel", &grpc_server_channel_trace); grpc_register_tracer("bdp_estimator", &grpc_bdp_estimator_trace); // Default pluck trace to 1 - grpc_cq_pluck_trace = 1; grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace); // Default timeout trace to 1 - grpc_cq_event_timeout_trace = 1; grpc_register_tracer("op_failure", &grpc_trace_operation_failures); grpc_register_tracer("resource_quota", &grpc_resource_quota_trace); grpc_register_tracer("call_error", &grpc_call_error_trace); @@ -173,6 +171,7 @@ void grpc_init(void) { grpc_tracer_init("GRPC_TRACE"); /* no more changes to channel init pipelines */ grpc_channel_init_finalize(); + grpc_iomgr_start(); } gpr_mu_unlock(&g_init_mu); GRPC_API_TRACE("grpc_init(void)", 0, ()); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 934ca0431a6..99020926c91 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -73,7 +73,7 @@ typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; -int grpc_server_channel_trace = 0; +grpc_tracer_flag grpc_server_channel_trace; typedef struct requested_call { requested_call_type type; @@ -456,7 +456,7 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand, grpc_closure_init(&chand->finish_destroy_channel_closure, finish_destroy_channel, chand, grpc_schedule_on_exec_ctx); - if (grpc_server_channel_trace && error != GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_server_channel_trace) && error != GRPC_ERROR_NONE) { const char *msg = grpc_error_string(error); gpr_log(GPR_INFO, "Disconnected client: %s", msg); } diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index a85d9f4964c..cd2fca0fe02 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -36,12 +36,13 @@ #include #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/transport/transport.h" extern const grpc_channel_filter grpc_server_top_filter; /** Lightweight tracing of server channel state */ -extern int grpc_server_channel_trace; +extern grpc_tracer_flag grpc_server_channel_trace; /* Add a listener to the server: when the server starts, it will call start, and when it shuts down, it will call destroy */ diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c index e1483677fd3..c4c0078cf28 100644 --- a/src/core/lib/transport/bdp_estimator.c +++ b/src/core/lib/transport/bdp_estimator.c @@ -38,7 +38,7 @@ #include #include -int grpc_bdp_estimator_trace = 0; +grpc_tracer_flag grpc_bdp_estimator_trace; void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) { estimator->estimate = 65536; @@ -67,7 +67,7 @@ bool grpc_bdp_estimator_add_incoming_bytes(grpc_bdp_estimator *estimator, } void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:sched acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } @@ -77,7 +77,7 @@ void grpc_bdp_estimator_schedule_ping(grpc_bdp_estimator *estimator) { } void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:start acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } @@ -87,14 +87,14 @@ void grpc_bdp_estimator_start_ping(grpc_bdp_estimator *estimator) { } void grpc_bdp_estimator_complete_ping(grpc_bdp_estimator *estimator) { - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]:complete acc=%" PRId64 " est=%" PRId64, estimator->name, estimator->accumulator, estimator->estimate); } GPR_ASSERT(estimator->ping_state == GRPC_BDP_PING_STARTED); if (estimator->accumulator > 2 * estimator->estimate / 3) { estimator->estimate *= 2; - if (grpc_bdp_estimator_trace) { + if (GRPC_TRACER_ON(grpc_bdp_estimator_trace)) { gpr_log(GPR_DEBUG, "bdp[%s]: estimate increased to %" PRId64, estimator->name, estimator->estimate); } diff --git a/src/core/lib/transport/bdp_estimator.h b/src/core/lib/transport/bdp_estimator.h index df8d1f6fc0f..1b125d8d611 100644 --- a/src/core/lib/transport/bdp_estimator.h +++ b/src/core/lib/transport/bdp_estimator.h @@ -36,11 +36,12 @@ #include #include +#include "src/core/lib/debug/trace.h" #define GRPC_BDP_SAMPLES 16 #define GRPC_BDP_MIN_SAMPLES_FOR_ESTIMATE 3 -extern int grpc_bdp_estimator_trace; +extern grpc_tracer_flag grpc_bdp_estimator_trace; typedef enum { GRPC_BDP_PING_UNSCHEDULED, diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 3757b252676..0b8dc703e2c 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -39,7 +39,7 @@ #include #include -int grpc_connectivity_state_trace = 0; +grpc_tracer_flag grpc_connectivity_state_trace; const char *grpc_connectivity_state_name(grpc_connectivity_state state) { switch (state) { @@ -94,7 +94,7 @@ grpc_connectivity_state grpc_connectivity_state_check( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); } @@ -106,7 +106,7 @@ grpc_connectivity_state grpc_connectivity_state_get( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: get %s", tracker, tracker->name, grpc_connectivity_state_name(cur)); } @@ -127,7 +127,7 @@ bool grpc_connectivity_state_notify_on_state_change( grpc_connectivity_state cur = (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { if (current == NULL) { gpr_log(GPR_DEBUG, "CONWATCH: %p %s: unsubscribe notify=%p", tracker, tracker->name, notify); @@ -180,7 +180,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, (grpc_connectivity_state)gpr_atm_no_barrier_load( &tracker->current_state_atm); grpc_connectivity_state_watcher *w; - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { const char *error_string = grpc_error_string(error); gpr_log(GPR_DEBUG, "SET: %p %s: %s --> %s [%s] error=%p %s", tracker, tracker->name, grpc_connectivity_state_name(cur), @@ -208,7 +208,7 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx, while ((w = tracker->watchers) != NULL) { *w->current = state; tracker->watchers = w->next; - if (grpc_connectivity_state_trace) { + if (GRPC_TRACER_ON(grpc_connectivity_state_trace)) { gpr_log(GPR_DEBUG, "NOTIFY: %p %s: %p", tracker, tracker->name, w->notify); } diff --git a/src/core/lib/transport/connectivity_state.h b/src/core/lib/transport/connectivity_state.h index c9604c34dda..cdc2930c11c 100644 --- a/src/core/lib/transport/connectivity_state.h +++ b/src/core/lib/transport/connectivity_state.h @@ -35,6 +35,7 @@ #define GRPC_CORE_LIB_TRANSPORT_CONNECTIVITY_STATE_H #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" typedef struct grpc_connectivity_state_watcher { @@ -57,7 +58,7 @@ typedef struct { char *name; } grpc_connectivity_state_tracker; -extern int grpc_connectivity_state_trace; +extern grpc_tracer_flag grpc_connectivity_state_trace; /** enum --> string conversion */ const char *grpc_connectivity_state_name(grpc_connectivity_state state); diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index 1836beefc42..4925d19f96e 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -396,7 +396,7 @@ static tsi_result fake_handshaker_get_bytes_to_send_to_peer( if (next_message_to_send > TSI_FAKE_HANDSHAKE_MESSAGE_MAX) { next_message_to_send = TSI_FAKE_HANDSHAKE_MESSAGE_MAX; } - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%s prepared %s.", impl->is_client ? "Client" : "Server", tsi_fake_handshake_message_to_string(impl->next_message_to_send)); @@ -408,7 +408,7 @@ static tsi_result fake_handshaker_get_bytes_to_send_to_peer( if (!impl->is_client && impl->next_message_to_send == TSI_FAKE_HANDSHAKE_MESSAGE_MAX) { /* We're done. */ - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "Server is done."); } impl->result = TSI_OK; @@ -445,7 +445,7 @@ static tsi_result fake_handshaker_process_bytes_from_peer( tsi_fake_handshake_message_to_string(received_msg), tsi_fake_handshake_message_to_string(expected_msg)); } - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%s received %s.", impl->is_client ? "Client" : "Server", tsi_fake_handshake_message_to_string(received_msg)); } @@ -453,7 +453,7 @@ static tsi_result fake_handshaker_process_bytes_from_peer( impl->needs_incoming_message = 0; if (impl->next_message_to_send == TSI_FAKE_HANDSHAKE_MESSAGE_MAX) { /* We're done. */ - if (tsi_tracing_enabled) { + if (GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%s is done.", impl->is_client ? "Client" : "Server"); } impl->result = TSI_OK; diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index e1d634a1fa4..59fd2b1c93a 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -180,7 +180,7 @@ static const char *ssl_error_string(int error) { /* TODO(jboeuf): Remove when we are past the debugging phase with this code. */ static void ssl_log_where_info(const SSL *ssl, int where, int flag, const char *msg) { - if ((where & flag) && tsi_tracing_enabled) { + if ((where & flag) && GRPC_TRACER_ON(tsi_tracing_enabled)) { gpr_log(GPR_INFO, "%20.20s - %30.30s - %5.10s", msg, SSL_state_string_long(ssl), SSL_state_string(ssl)); } diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index b11c00c43c4..9a2d4669c67 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -41,7 +41,7 @@ /* --- Tracing. --- */ -int tsi_tracing_enabled = 0; +grpc_tracer_flag tsi_tracing_enabled; /* --- tsi_result common implementation. --- */ diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index a4c9cbc0011..2422f920767 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -36,13 +36,14 @@ #include +#include "src/core/lib/debug/trace.h" #include "src/core/tsi/transport_security_interface.h" #ifdef __cplusplus extern "C" { #endif -extern int tsi_tracing_enabled; +extern grpc_tracer_flag tsi_tracing_enabled; /* Base for tsi_frame_protector implementations. See transport_security_interface.h for documentation. */ diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index f2112b62b67..8a3fff6a170 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -37,6 +37,8 @@ #include #include +#include "src/core/lib/debug/trace.h" + #ifdef __cplusplus extern "C" { #endif @@ -73,8 +75,7 @@ const char *tsi_result_to_string(tsi_result result); /* --- tsi tracing --- */ -/* Set this early to avoid races */ -extern int tsi_tracing_enabled; +extern grpc_tracer_flag tsi_tracing_enabled; /* --- tsi_frame_protector object --- diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index 88a0e301da1..b33b43dac58 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -44,6 +44,7 @@ #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/tcp_client.h" #include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/timer_manager.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/transport/metadata.h" @@ -722,6 +723,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { grpc_tcp_client_connect_impl = my_tcp_client_connect; gpr_now_impl = now_impl; grpc_init(); + grpc_timer_manager_set_threading(false); grpc_resolve_address = my_resolve_address; GPR_ASSERT(g_channel == NULL); @@ -769,6 +771,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { g_now = gpr_time_add(g_now, gpr_time_from_seconds(1, GPR_TIMESPAN)); } + grpc_timer_manager_tick(); + switch (next_byte(&inp)) { // terminate on bad bytes default: diff --git a/test/core/transport/connectivity_state_test.c b/test/core/transport/connectivity_state_test.c index 8314a5f6190..96db59ba2df 100644 --- a/test/core/transport/connectivity_state_test.c +++ b/test/core/transport/connectivity_state_test.c @@ -151,7 +151,7 @@ static void test_subscribe_with_failure_then_destroy(void) { int main(int argc, char **argv) { grpc_test_init(argc, argv); - grpc_connectivity_state_trace = 1; + grpc_connectivity_state_trace.value = 1; test_connectivity_state_name(); test_check(); test_subscribe_then_unsubscribe(); From f3003885911eb303db69bcfef98161b8326b7dae Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 3 May 2017 13:56:15 -0700 Subject: [PATCH 117/177] Better? feature test --- src/core/lib/debug/trace.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/lib/debug/trace.h b/src/core/lib/debug/trace.h index bf3bfee306f..ba432574d06 100644 --- a/src/core/lib/debug/trace.h +++ b/src/core/lib/debug/trace.h @@ -38,9 +38,11 @@ #include #include -#if defined(__has_feature) && __has_feature(thread_sanitizer) +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) #define GRPC_THREADSAFE_TRACER #endif +#endif typedef struct { #ifdef GRPC_THREADSAFE_TRACER From 9f276e19a8ea1cbf5d445ed44635ebb6142c62ec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 3 May 2017 14:07:10 -0700 Subject: [PATCH 118/177] Fix mac build --- src/core/lib/surface/api_trace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 3b20e042b47..55c69cd5b0b 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -31,6 +31,7 @@ * */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/surface/api_trace.h" -grpc_tracer_flag grpc_api_trace; +grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false); From 4deb397d9fd74d7d8e5c4064b5cf43f618c4a912 Mon Sep 17 00:00:00 2001 From: MakMukhi Date: Wed, 3 May 2017 17:26:14 -0700 Subject: [PATCH 119/177] Create unit_testing.md --- doc/unit_testing.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/unit_testing.md diff --git a/doc/unit_testing.md b/doc/unit_testing.md new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/doc/unit_testing.md @@ -0,0 +1 @@ + From 7431a78628ffcd86fd7e7512610ad118c16eb6f1 Mon Sep 17 00:00:00 2001 From: MakMukhi Date: Wed, 3 May 2017 17:50:31 -0700 Subject: [PATCH 120/177] How to write unit tests for client sync APIs. --- doc/unit_testing.md | 174 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/doc/unit_testing.md b/doc/unit_testing.md index 8b137891791..3307d115879 100644 --- a/doc/unit_testing.md +++ b/doc/unit_testing.md @@ -1 +1,175 @@ +# How to write unit tests for gRPC C++ client. + +tl;dr: [Example code](https://github.com/grpc/grpc/blob/master/test/cpp/end2end/mock_test.cc). + +To unit-test client-side logic via the synchronous API, gRPC provides a mocked Stub that can be programmed upon and easily incorporated in the test code. + +For instance, consider an EchoService like this: + + +```proto +service EchoTestService { +        rpc Echo(EchoRequest) returns (EchoResponse); +        rpc BidiStream(stream EchoRequest) returns (stream EchoResponse); +} +``` + +The code generate would look something like this: + +```c++ +class EchoTestService final { + public: + class StubInterface { + virtual ::grpc::Status Echo(::grpc::ClientContext* context, const ::grpc::testing::EchoRequest& request, ::grpc::testing::EchoResponse* response) = 0; + … + std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>> BidiStream(::grpc::ClientContext* context) { + return std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>>(BidiStreamRaw(context)); + } + … + private: + virtual ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>* BidiStreamRaw(::grpc::ClientContext* context) = 0; + … +  } // End StubInterface +… +} // End EchoTestService +``` + + +If we mock the StubInterface and set expectations on the pure-virtual methods we can test client-side logic without having to make any rpcs. + +A mock for this StubInterface will look like this: + + +```c++ +class MockEchoTestServiceStub : public EchoTestService::StubInterface { + public: + MOCK_METHOD3(Echo, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::EchoRequest& request, ::grpc::testing::EchoResponse* response)); + MOCK_METHOD1(BidiStreamRaw, ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>*(::grpc::ClientContext* context)); +}; +``` + + +**Generating mock code:** + +Such a mock can be auto-generated by: + + + +1. Setting flag(generate_mock_code=true) on grpc plugin for protoc, or +1. Setting an attribute(generate_mock) in your bazel rule. + +Protoc plugin flag: + +```sh +protoc -I . --grpc_out=generate_mock_code=true:. --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` echo.proto +``` + +Bazel rule: + +```py +grpc_proto_library( + name = "echo_proto", + srcs = ["echo.proto"], + generate_mock = True, +) +``` + + +By adding such a flag now a header file `echo_mock.h` containing the mocked stub will also be generated. + +This header file can then be included in test files along with a gmock dependency. + +**Writing tests with mocked Stub.** + +Consider the following client a user might have: + +```c++ +class FakeClient { + public: + explicit FakeClient(EchoTestService::StubInterface* stub) : stub_(stub) {} + + void DoEcho() { + ClientContext context; + EchoRequest request; + EchoResponse response; + request.set_message("hello world"); + Status s = stub_->Echo(&context, request, &response); + EXPECT_EQ(request.message(), response.message()); + EXPECT_TRUE(s.ok()); + } + + void DoBidiStream() { + EchoRequest request; + EchoResponse response; + ClientContext context; + grpc::string msg("hello"); + + std::unique_ptr> + stream = stub_->BidiStream(&context); + + request.set_message(msg + "0"); + EXPECT_TRUE(stream->Write(request)); + EXPECT_TRUE(stream->Read(&response)); + EXPECT_EQ(response.message(), request.message()); + + request.set_message(msg + "1"); + EXPECT_TRUE(stream->Write(request)); + EXPECT_TRUE(stream->Read(&response)); + EXPECT_EQ(response.message(), request.message()); + + request.set_message(msg + "2"); + EXPECT_TRUE(stream->Write(request)); + EXPECT_TRUE(stream->Read(&response)); + EXPECT_EQ(response.message(), request.message()); + + stream->WritesDone(); + EXPECT_FALSE(stream->Read(&response)); + + Status s = stream->Finish(); + EXPECT_TRUE(s.ok()); + } + + void ResetStub(EchoTestService::StubInterface* stub) { stub_ = stub; } + + private: + EchoTestService::StubInterface* stub_; +}; +``` + +A test could initialize this FakeClient with a mocked stub having set expectations on it: + +Unary RPC: + +```c++ +MockEchoTestServiceStub stub; +EchoResponse resp; +resp.set_message("hello world"); +Expect_CALL(stub, Echo(_,_,_)).Times(Atleast(1)).WillOnce(DoAll(SetArgPointee<2>(resp), Return(Status::OK))); +FakeClient client(stub); +client.DoEcho(); +``` + +Streaming RPC: + +```c++ +ACTION_P(copy, msg) { + arg0->set_message(msg->message()); +} + + +auto rw = new MockClientReaderWriter(); +EchoRequest msg; +EXPECT_CALL(*rw, Write(_, _)).Times(3).WillRepeatedly(DoAll(SaveArg<0>(&msg), Return(true))); +EXPECT_CALL(*rw, Read(_)). + WillOnce(DoAll(WithArg<0>(copy(&msg)), Return(true))). + WillOnce(DoAll(WithArg<0>(copy(&msg)), Return(true))). + WillOnce(DoAll(WithArg<0>(copy(&msg)), Return(true))). + WillOnce(Return(false)); + +MockEchoTestServiceStub stub; +EXPECT_CALL(stub, BidiStreamRaw(_)).Times(AtLeast(1)).WillOnce(Return(rw)); + +FakeClient client(stub); +client.DoBidiStream(); +``` From c0a9d1f4252763fe0a37ad9cb6046918ce94a034 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Wed, 3 May 2017 18:24:06 -0700 Subject: [PATCH 121/177] Turnstile polling per dedicated epoll set --- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index d11dbbfae8c..8679f9c85d7 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -110,7 +110,7 @@ static void fd_global_init(void); static void fd_global_shutdown(void); /******************************************************************************* - * Polling island Declarations + * epoll set Declarations */ #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG @@ -130,6 +130,10 @@ static void fd_global_shutdown(void); typedef struct epoll_set { grpc_closure_scheduler workqueue_scheduler; + /* Mutex poller should acquire to poll this. This enforces that only one + * poller can be polling on epoll_set at any time */ + gpr_mu mu; + /* Ref count. Use EPS_ADD_REF() and EPS_UNREF() macros to increment/decrement the refcount. Once the ref count becomes zero, this structure is destroyed which means we should ensure that there is never a scenario where a @@ -137,7 +141,7 @@ typedef struct epoll_set { zero. */ gpr_atm ref_count; - /* Number of threads currently polling on this island */ + /* Number of threads currently polling on this epoll set*/ gpr_atm poller_count; /* Mutex guarding the read end of the workqueue (must be held to pop from * workqueue_items) */ @@ -189,6 +193,7 @@ struct grpc_pollset_set {}; size_t g_num_eps = 1; struct epoll_set **g_epoll_sets = NULL; +gpr_atm g_next_eps; size_t g_num_threads_per_eps = 1; gpr_thd_id *g_poller_threads = NULL; @@ -219,16 +224,13 @@ static bool append_error(grpc_error **composite, grpc_error *error, } /******************************************************************************* - * Polling island Definitions + * epoll set Definitions */ -/* The wakeup fd that is used to wake up all threads in a Polling island. This - is useful in the epoll set merge operation where we need to wakeup all - the threads currently polling the smaller epoll set (so that they can - start polling the new/merged epoll set) - - NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the - threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +/* The wakeup fd that is used to wake up all threads in an epoll_set informing + that the epoll set is shutdown. This wakeup fd initialized to be readable + and MUST NOT be consumed i.e the threads that woke up MUST NOT call + grpc_wakeup_fd_consume_wakeup() */ static grpc_wakeup_fd epoll_set_wakeup_fd; /* The epoll set being polled right now. @@ -399,6 +401,7 @@ static epoll_set *epoll_set_create(grpc_error **error) { eps->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; eps->epoll_fd = -1; + gpr_mu_init(&eps->mu); gpr_mu_init(&eps->workqueue_read_mu); gpr_mpscq_init(&eps->workqueue_items); gpr_atm_rel_store(&eps->workqueue_item_count, 0); @@ -437,6 +440,7 @@ static void epoll_set_delete(epoll_set *eps) { } GPR_ASSERT(gpr_atm_no_barrier_load(&eps->workqueue_item_count) == 0); + gpr_mu_destroy(&eps->mu); gpr_mu_destroy(&eps->workqueue_read_mu); gpr_mpscq_destroy(&eps->workqueue_items); grpc_wakeup_fd_destroy(&eps->workqueue_wakeup_fd); @@ -897,6 +901,19 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, epoll_set *eps) { return false; } +/* Blocking call */ +static void acquire_epoll_lease(epoll_set *eps) { + if (g_num_threads_per_eps > 1) { + gpr_mu_lock(&eps->mu); + } +} + +static void release_epoll_lease(epoll_set *eps) { + if (g_num_threads_per_eps > 1) { + gpr_mu_unlock(&eps->mu); + } +} + #define GRPC_EPOLL_MAX_EVENTS 100 static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, grpc_error **error) { @@ -908,7 +925,9 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, int timeout_ms = -1; GRPC_SCHEDULING_START_BLOCKING_REGION; + acquire_epoll_lease(eps); ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms); + release_epoll_lease(eps); GRPC_SCHEDULING_END_BLOCKING_REGION; if (ep_rv < 0) { @@ -961,11 +980,6 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, epoll set. */ epoll_fd = eps->epoll_fd; - /* Add an extra ref so that the island does not get destroyed (which means - the epoll_fd won't be closed) while we are are doing an epoll_wait() on the - epoll_fd */ - EPS_ADD_REF(eps, "ps_work"); - /* If we get some workqueue work to do, it might end up completing an item on the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ @@ -979,13 +993,6 @@ static void epoll_set_work(grpc_exec_ctx *exec_ctx, epoll_set *eps, gpr_atm_no_barrier_fetch_add(&eps->poller_count, -1); } - /* Before leaving, release the extra ref we added to the epoll set. It - is important to use "eps" here (i.e our old copy of pollset->eps - that we got before releasing the epoll set lock). This is because - pollset->eps pointer might get udpated in other parts of the - code when there is an island merge while we are doing epoll_wait() above */ - EPS_UNREF(exec_ctx, eps, "ps_work"); - GPR_TIMER_END("epoll_set_work", 0); } @@ -1162,7 +1169,7 @@ static void add_fd_to_eps(grpc_fd *fd) { GPR_TIMER_BEGIN("add_fd_to_eps", 0); grpc_error *error = GRPC_ERROR_NONE; - size_t idx = ((size_t)rand()) % g_num_eps; + size_t idx = (size_t)gpr_atm_no_barrier_fetch_add(&g_next_eps, 1) % g_num_eps; epoll_set *eps = g_epoll_sets[idx]; gpr_mu_lock(&fd->mu); @@ -1176,8 +1183,7 @@ static void add_fd_to_eps(grpc_fd *fd) { EPS_ADD_REF(eps, "fd"); fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, - idx); + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, idx); gpr_mu_unlock(&fd->mu); GRPC_LOG_IF_ERROR("add_fd_to_eps", error); @@ -1203,6 +1209,7 @@ static bool init_epoll_sets() { EPS_ADD_REF(g_epoll_sets[i], "init_epoll_sets"); } + gpr_atm_no_barrier_store(&g_next_eps, 0); gpr_mu *mu; pollset_init(&g_read_notifier, &mu); @@ -1247,14 +1254,14 @@ static void start_poller_threads() { gpr_log(GPR_INFO, "Starting poller threads"); - /* One thread per pollset */ - g_poller_threads = (gpr_thd_id *)malloc(g_num_eps * sizeof(gpr_thd_id)); + size_t num_threads = g_num_eps * g_num_threads_per_eps; + g_poller_threads = (gpr_thd_id *)malloc(num_threads * sizeof(gpr_thd_id)); gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); - for (size_t i = 0; i < g_num_eps; i++) { + for (size_t i = 0; i < num_threads; i++) { gpr_thd_new(&g_poller_threads[i], poller_thread_loop, - (void *)g_epoll_sets[i], &options); + (void *)g_epoll_sets[i % g_num_eps], &options); } } @@ -1266,7 +1273,8 @@ static void shutdown_poller_threads() { gpr_log(GPR_INFO, "Shutting down pollers"); epoll_set *eps = NULL; - for (size_t i = 0; i < g_num_eps; i++) { + size_t num_threads = g_num_eps * g_num_threads_per_eps; + for (size_t i = 0; i < num_threads; i++) { eps = g_epoll_sets[i]; epoll_set_add_wakeup_fd_locked(eps, &epoll_set_wakeup_fd, &error); } From 3b654361537dee42dba9855c7d2501302c272c7a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 08:11:17 -0700 Subject: [PATCH 122/177] Add initializers for all tracers --- .../ext/filters/client_channel/lb_policy/grpclb/grpclb.c | 2 +- .../client_channel/lb_policy/round_robin/round_robin.c | 2 +- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 7 ++++--- src/core/lib/channel/channel_stack.c | 2 +- src/core/lib/channel/channel_stack_builder.c | 3 ++- src/core/lib/http/parser.c | 2 +- src/core/lib/iomgr/combiner.c | 2 +- src/core/lib/iomgr/resource_quota.c | 5 +++-- src/core/lib/iomgr/tcp_posix.c | 2 +- src/core/lib/iomgr/timer_generic.c | 4 ++-- src/core/lib/security/transport/secure_endpoint.c | 2 +- src/core/lib/surface/call.c | 4 ++-- src/core/lib/surface/completion_queue.c | 4 ++-- src/core/lib/surface/server.c | 2 +- src/core/lib/transport/bdp_estimator.c | 2 +- src/core/lib/transport/connectivity_state.c | 2 +- src/core/tsi/transport_security.c | 2 +- 17 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index a15476dc23e..18b36dde8a1 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -138,7 +138,7 @@ #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 -grpc_tracer_flag grpc_lb_glb_trace; +grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false); /* add lb_token of selected subchannel (address) to the call's initial * metadata */ diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c index 075b7caf5c7..6e7f4106358 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c @@ -74,7 +74,7 @@ typedef struct round_robin_lb_policy round_robin_lb_policy; -grpc_tracer_flag grpc_lb_round_robin_trace; +grpc_tracer_flag grpc_lb_round_robin_trace = GRPC_TRACER_INITIALIZER(false); /** List of entities waiting for a pick. * diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index c5c0ec316d8..ab74ff6ad31 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -89,8 +89,8 @@ static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; #define MAX_CLIENT_STREAM_ID 0x7fffffffu -grpc_tracer_flag grpc_http_trace; -grpc_tracer_flag grpc_flowctl_trace; +grpc_tracer_flag grpc_http_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_flowctl_trace = GRPC_TRACER_INITIALIZER(false); static const grpc_transport_vtable vtable; @@ -2787,7 +2787,8 @@ static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error_set_int( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Buffers full"), GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_ENHANCE_YOUR_CALM)); - } else if (error == GRPC_ERROR_NONE && GRPC_TRACER_ON(grpc_resource_quota_trace)) { + } else if (error == GRPC_ERROR_NONE && + GRPC_TRACER_ON(grpc_resource_quota_trace)) { gpr_log(GPR_DEBUG, "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR " streams", diff --git a/src/core/lib/channel/channel_stack.c b/src/core/lib/channel/channel_stack.c index 107d5997fdf..7db54d11074 100644 --- a/src/core/lib/channel/channel_stack.c +++ b/src/core/lib/channel/channel_stack.c @@ -38,7 +38,7 @@ #include #include -grpc_tracer_flag grpc_trace_channel; +grpc_tracer_flag grpc_trace_channel = GRPC_TRACER_INITIALIZER(false); /* Memory layouts. diff --git a/src/core/lib/channel/channel_stack_builder.c b/src/core/lib/channel/channel_stack_builder.c index 332635f58fb..44b030e4d1f 100644 --- a/src/core/lib/channel/channel_stack_builder.c +++ b/src/core/lib/channel/channel_stack_builder.c @@ -38,7 +38,8 @@ #include #include -grpc_tracer_flag grpc_trace_channel_stack_builder; +grpc_tracer_flag grpc_trace_channel_stack_builder = + GRPC_TRACER_INITIALIZER(false); typedef struct filter_node { struct filter_node *next; diff --git a/src/core/lib/http/parser.c b/src/core/lib/http/parser.c index b8883f5953e..a4357978e43 100644 --- a/src/core/lib/http/parser.c +++ b/src/core/lib/http/parser.c @@ -40,7 +40,7 @@ #include #include -grpc_tracer_flag grpc_http1_trace; +grpc_tracer_flag grpc_http1_trace = GRPC_TRACER_INITIALIZER(false); static char *buf2str(void *buffer, size_t length) { char *out = gpr_malloc(length + 1); diff --git a/src/core/lib/iomgr/combiner.c b/src/core/lib/iomgr/combiner.c index fa7d576133f..863f22c6141 100644 --- a/src/core/lib/iomgr/combiner.c +++ b/src/core/lib/iomgr/combiner.c @@ -42,7 +42,7 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" -grpc_tracer_flag grpc_combiner_trace; +grpc_tracer_flag grpc_combiner_trace = GRPC_TRACER_INITIALIZER(false); #define GRPC_COMBINER_TRACE(fn) \ do { \ diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 52ccfc0aad1..6b2b85cce00 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -44,7 +44,7 @@ #include "src/core/lib/iomgr/combiner.h" -grpc_tracer_flag grpc_resource_quota_trace; +grpc_tracer_flag grpc_resource_quota_trace = GRPC_TRACER_INITIALIZER(false); #define MEMORY_USAGE_ESTIMATION_MAX 65536 @@ -313,7 +313,8 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, resource_quota->name, resource_user->name, amt, resource_quota->free_pool); } - } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && resource_user->free_pool >= 0) { + } else if (GRPC_TRACER_ON(grpc_resource_quota_trace) && + resource_user->free_pool >= 0) { gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request", resource_quota->name, resource_user->name); } diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 233dea9bece..5d360b0b805 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -74,7 +74,7 @@ typedef GRPC_MSG_IOVLEN_TYPE msg_iovlen_type; typedef size_t msg_iovlen_type; #endif -grpc_tracer_flag grpc_tcp_trace; +grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { grpc_endpoint base; diff --git a/src/core/lib/iomgr/timer_generic.c b/src/core/lib/iomgr/timer_generic.c index ec0bb169cb3..b28340b71c7 100644 --- a/src/core/lib/iomgr/timer_generic.c +++ b/src/core/lib/iomgr/timer_generic.c @@ -56,8 +56,8 @@ #define MIN_QUEUE_WINDOW_DURATION 0.01 #define MAX_QUEUE_WINDOW_DURATION 1 -grpc_tracer_flag grpc_timer_trace; -grpc_tracer_flag grpc_timer_check_trace; +grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { gpr_mu mu; diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index c8dc50206ae..48d368a2a70 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -75,7 +75,7 @@ typedef struct { gpr_refcount ref; } secure_endpoint; -grpc_tracer_flag grpc_trace_secure_endpoint; +grpc_tracer_flag grpc_trace_secure_endpoint = GRPC_TRACER_INITIALIZER(false); static void destroy(grpc_exec_ctx *exec_ctx, secure_endpoint *secure_ep) { secure_endpoint *ep = secure_ep; diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index fbced008603..1bd158f614b 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -244,8 +244,8 @@ struct grpc_call { void *saved_receiving_stream_ready_bctlp; }; -grpc_tracer_flag grpc_call_error_trace; -grpc_tracer_flag grpc_compression_trace; +grpc_tracer_flag grpc_call_error_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_compression_trace = GRPC_TRACER_INITIALIZER(false); #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index f31f0bc4c7c..f5ce96f62d8 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -50,9 +50,9 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/event_string.h" -grpc_tracer_flag grpc_trace_operation_failures; +grpc_tracer_flag grpc_trace_operation_failures = GRPC_TRACER_INITIALIZER(false); #ifndef NDEBUG -grpc_tracer_flag grpc_trace_pending_tags; +grpc_tracer_flag grpc_trace_pending_tags = GRPC_TRACER_INITIALIZER(false); #endif typedef struct { diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 99020926c91..795429ed1e4 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -73,7 +73,7 @@ typedef struct registered_method registered_method; typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type; -grpc_tracer_flag grpc_server_channel_trace; +grpc_tracer_flag grpc_server_channel_trace = GRPC_TRACER_INITIALIZER(false); typedef struct requested_call { requested_call_type type; diff --git a/src/core/lib/transport/bdp_estimator.c b/src/core/lib/transport/bdp_estimator.c index c4c0078cf28..eaf9caaa645 100644 --- a/src/core/lib/transport/bdp_estimator.c +++ b/src/core/lib/transport/bdp_estimator.c @@ -38,7 +38,7 @@ #include #include -grpc_tracer_flag grpc_bdp_estimator_trace; +grpc_tracer_flag grpc_bdp_estimator_trace = GRPC_TRACER_INITIALIZER(false); void grpc_bdp_estimator_init(grpc_bdp_estimator *estimator, const char *name) { estimator->estimate = 65536; diff --git a/src/core/lib/transport/connectivity_state.c b/src/core/lib/transport/connectivity_state.c index 0b8dc703e2c..e30cd523fa4 100644 --- a/src/core/lib/transport/connectivity_state.c +++ b/src/core/lib/transport/connectivity_state.c @@ -39,7 +39,7 @@ #include #include -grpc_tracer_flag grpc_connectivity_state_trace; +grpc_tracer_flag grpc_connectivity_state_trace = GRPC_TRACER_INITIALIZER(false); const char *grpc_connectivity_state_name(grpc_connectivity_state state) { switch (state) { diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index 9a2d4669c67..4efcf8f43d8 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -41,7 +41,7 @@ /* --- Tracing. --- */ -grpc_tracer_flag tsi_tracing_enabled; +grpc_tracer_flag tsi_tracing_enabled = GRPC_TRACER_INITIALIZER(false); /* --- tsi_result common implementation. --- */ From 9ba9f704c30640a6cd98195b4eaf4b1fab660bec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 08:17:30 -0700 Subject: [PATCH 123/177] Fix race --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index ab74ff6ad31..654ff0c09dd 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1485,7 +1485,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (GRPC_TRACER_ON(grpc_http_trace)) { char *str = grpc_transport_stream_op_batch_string(op); - gpr_log(GPR_DEBUG, "perform_stream_op[s=%p/%d]: %s", s, s->id, str); + gpr_log(GPR_DEBUG, "perform_stream_op[s=%p]: %s", s, str); gpr_free(str); } From 1dbd063dbf08cd39ab398c75c1c458ee6bcf4993 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 16:48:32 -0700 Subject: [PATCH 124/177] Fix old trace vars --- .../ext/filters/http/server/http_server_filter.c | 2 -- .../ext/transport/chttp2/transport/hpack_encoder.c | 6 +++--- .../ext/transport/chttp2/transport/hpack_table.c | 7 ++++--- src/core/lib/iomgr/tcp_client_uv.c | 6 +++--- src/core/lib/iomgr/tcp_uv.h | 3 ++- test/core/iomgr/timer_list_test.c | 13 +++++++------ 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/core/ext/filters/http/server/http_server_filter.c b/src/core/ext/filters/http/server/http_server_filter.c index ff857878e46..9e495f4d422 100644 --- a/src/core/ext/filters/http/server/http_server_filter.c +++ b/src/core/ext/filters/http/server/http_server_filter.c @@ -46,8 +46,6 @@ #define EXPECTED_CONTENT_TYPE "application/grpc" #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1 -extern int grpc_http_trace; - typedef struct call_data { grpc_linked_mdelem status; grpc_linked_mdelem content_type; diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 8fdd4ee77c1..126e012aac9 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -69,7 +69,7 @@ static grpc_slice_refcount terminal_slice_refcount = {NULL, NULL}; static const grpc_slice terminal_slice = {&terminal_slice_refcount, .data.refcounted = {0, 0}}; -extern int grpc_http_trace; +extern grpc_tracer_flag grpc_http_trace; typedef struct { int is_first_frame; @@ -425,7 +425,7 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, "Reserved header (colon-prefixed) happening after regular ones."); } - if (grpc_http_trace && !GRPC_MDELEM_IS_INTERNED(elem)) { + if (GRPC_TRACER_ON(grpc_http_trace) && !GRPC_MDELEM_IS_INTERNED(elem)) { char *k = grpc_slice_to_c_string(GRPC_MDKEY(elem)); char *v = grpc_slice_to_c_string(GRPC_MDVALUE(elem)); gpr_log( @@ -616,7 +616,7 @@ void grpc_chttp2_hpack_compressor_set_max_table_size( } } c->advertise_table_size_change = 1; - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "set max table size from encoder to %d", max_table_size); } } diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c index 9dd41fdbe16..7aaff553396 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.c +++ b/src/core/ext/transport/chttp2/transport/hpack_table.c @@ -40,9 +40,10 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/support/murmur_hash.h" -extern int grpc_http_trace; +extern grpc_tracer_flag grpc_http_trace; static struct { const char *key; @@ -260,7 +261,7 @@ void grpc_chttp2_hptbl_set_max_bytes(grpc_exec_ctx *exec_ctx, if (tbl->max_bytes == max_bytes) { return; } - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "Update hpack parser max size to %d", max_bytes); } while (tbl->mem_used > max_bytes) { @@ -284,7 +285,7 @@ grpc_error *grpc_chttp2_hptbl_set_current_table_size(grpc_exec_ctx *exec_ctx, gpr_free(msg); return err; } - if (grpc_http_trace) { + if (GRPC_TRACER_ON(grpc_http_trace)) { gpr_log(GPR_DEBUG, "Update hpack parser table size to %d", bytes); } while (tbl->mem_used > bytes) { diff --git a/src/core/lib/iomgr/tcp_client_uv.c b/src/core/lib/iomgr/tcp_client_uv.c index 682c24ed56b..f0856a76d4b 100644 --- a/src/core/lib/iomgr/tcp_client_uv.c +++ b/src/core/lib/iomgr/tcp_client_uv.c @@ -46,7 +46,7 @@ #include "src/core/lib/iomgr/tcp_uv.h" #include "src/core/lib/iomgr/timer.h" -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; typedef struct grpc_uv_tcp_connect { uv_connect_t connect_req; @@ -72,7 +72,7 @@ static void uv_tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { int done; grpc_uv_tcp_connect *connect = acp; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: on_alarm: error=%s", connect->addr_name, str); @@ -156,7 +156,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, uv_tcp_init(uv_default_loop(), connect->tcp_handle); connect->connect_req.data = connect; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", connect->addr_name); } diff --git a/src/core/lib/iomgr/tcp_uv.h b/src/core/lib/iomgr/tcp_uv.h index 970fcafe4a5..106bec5eca7 100644 --- a/src/core/lib/iomgr/tcp_uv.h +++ b/src/core/lib/iomgr/tcp_uv.h @@ -44,11 +44,12 @@ otherwise specified. */ +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/endpoint.h" #include -extern int grpc_tcp_trace; +extern grpc_tracer_flag grpc_tcp_trace; #define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 diff --git a/test/core/iomgr/timer_list_test.c b/test/core/iomgr/timer_list_test.c index 46e41dd4490..88a9f6b8556 100644 --- a/test/core/iomgr/timer_list_test.c +++ b/test/core/iomgr/timer_list_test.c @@ -41,12 +41,13 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "test/core/util/test_config.h" #define MAX_CB 30 -extern int grpc_timer_trace; -extern int grpc_timer_check_trace; +extern grpc_tracer_flag grpc_timer_trace; +extern grpc_tracer_flag grpc_timer_check_trace; static int cb_called[MAX_CB][2]; @@ -63,8 +64,8 @@ static void add_test(void) { gpr_log(GPR_INFO, "add_test"); grpc_timer_list_init(start); - grpc_timer_trace = 1; - grpc_timer_check_trace = 1; + grpc_timer_trace.value = 1; + grpc_timer_check_trace.value = 1; memset(cb_called, 0, sizeof(cb_called)); /* 10 ms timers. will expire in the current epoch */ @@ -138,8 +139,8 @@ void destruction_test(void) { gpr_log(GPR_INFO, "destruction_test"); grpc_timer_list_init(gpr_time_0(GPR_CLOCK_REALTIME)); - grpc_timer_trace = 1; - grpc_timer_check_trace = 1; + grpc_timer_trace.value = 1; + grpc_timer_check_trace.value = 1; memset(cb_called, 0, sizeof(cb_called)); grpc_timer_init( From 9fe1bb1bc8bfca4678019438aa3e7ee0f69fcdfa Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 20:20:17 -0700 Subject: [PATCH 125/177] Fix BUILD --- BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/BUILD b/BUILD index 62f0e1d2c56..055e829be7c 100644 --- a/BUILD +++ b/BUILD @@ -510,6 +510,7 @@ grpc_cc_library( "src/core/lib/iomgr/tcp_windows.c", "src/core/lib/iomgr/time_averaged_stats.c", "src/core/lib/iomgr/timer_generic.c", + "src/core/lib/iomgr/timer_manager.c", "src/core/lib/iomgr/timer_heap.c", "src/core/lib/iomgr/timer_uv.c", "src/core/lib/iomgr/udp_server.c", @@ -627,6 +628,7 @@ grpc_cc_library( "src/core/lib/iomgr/time_averaged_stats.h", "src/core/lib/iomgr/timer.h", "src/core/lib/iomgr/timer_generic.h", + "src/core/lib/iomgr/timer_manager.h", "src/core/lib/iomgr/timer_heap.h", "src/core/lib/iomgr/timer_uv.h", "src/core/lib/iomgr/udp_server.h", From 14a7f9a2a834c6429309a44312ae089711ddedac Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 21:16:03 -0700 Subject: [PATCH 126/177] Reduce wakeups, comment code --- src/core/lib/iomgr/timer_manager.c | 73 ++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 1d83341299e..8b83cc0106c 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -52,8 +52,10 @@ static int g_thread_count; static int g_waiter_count; static completed_thread *g_completed_threads; static bool g_kicked; +static bool g_has_timed_waiter; +static uint64_t g_timed_waiter_generation; -#define MAX_WAITERS 3 +#define MAX_WAITERS 2 static void timer_thread(void *unused); @@ -92,39 +94,87 @@ void grpc_timer_manager_tick() { } static void timer_thread(void *unused) { + // this threads exec_ctx: we try to run things through to completion here + // since it's easy to spin up new threads grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); + const gpr_timespec inf_future = gpr_inf_future(GPR_CLOCK_MONOTONIC); for (;;) { - gpr_timespec next = gpr_inf_future(GPR_CLOCK_MONOTONIC); + gpr_timespec next = inf_future; gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + // check timer state, updates next to the next time to run a check if (grpc_timer_check(&exec_ctx, now, &next)) { + // if there's something to execute... gpr_mu_lock(&g_mu); + // remove a waiter from the pool, and start another thread if necessary --g_waiter_count; bool start_thread = g_waiter_count == 0; if (start_thread && g_threaded) { start_timer_thread_and_unlock(); } else { + // if there's no thread waiting with a timeout, kick an existing waiter + // so that the next deadline is not missed + if (!g_has_timed_waiter) { + gpr_log(GPR_DEBUG, "kick untimed waiter"); + gpr_cv_signal(&g_cv_wait); + } gpr_mu_unlock(&g_mu); } + // without our lock, flush the exec_ctx grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(&g_mu); + // garbage collect any threads hanging out that are dead gc_completed_threads(); + // get ready to wait again ++g_waiter_count; gpr_mu_unlock(&g_mu); } else { gpr_mu_lock(&g_mu); + // if we're not threaded anymore, leave if (!g_threaded) break; - if (gpr_cv_wait(&g_cv_wait, &g_mu, next)) { - if (g_kicked) { - grpc_timer_consume_kick(); - g_kicked = false; - } else if (g_waiter_count > MAX_WAITERS) { - break; - } + // if there's no timed waiter, we should become one: that waiter waits + // only until the next timer should expire + // all other timers wait forever + uint64_t my_timed_waiter_generation = g_timed_waiter_generation - 1; + if (!g_has_timed_waiter) { + g_has_timed_waiter = true; + // we use a generation counter to track the timed waiter so we can + // cancel an existing one quickly (and when it actually times out it'll + // figure stuff out instead of incurring a wakeup) + my_timed_waiter_generation = ++g_timed_waiter_generation; + gpr_log(GPR_DEBUG, "sleep for a while"); + } else { + next = inf_future; + gpr_log(GPR_DEBUG, "sleep until kicked"); + } + bool timed_out = gpr_cv_wait(&g_cv_wait, &g_mu, next); + // if we timed out and we have too many waiters, maybe exit this thread + bool should_stop = (timed_out && g_waiter_count > MAX_WAITERS); + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d timed_out:%d kicked:%d", + my_timed_waiter_generation == g_timed_waiter_generation, + timed_out, g_kicked); + // if this was the timed waiter, then we need to check timers, and flag + // that there's now no timed waiter... we'll look for a replacement if + // there's work to do after checking timers (code above) + if (my_timed_waiter_generation == g_timed_waiter_generation) { + g_has_timed_waiter = false; + should_stop = false; + } + // if this was a kick from the timer system, consume it (and don't stop + // this thread yet) + if (g_kicked) { + grpc_timer_consume_kick(); + g_kicked = false; + should_stop = false; + } + if (should_stop) { + break; } - gpr_mu_unlock(&g_mu); } + gpr_mu_unlock(&g_mu); } + // terminate the thread: drop the waiter count, thread count, and let whomever + // stopped the threading stuff know that we're done --g_waiter_count; --g_thread_count; if (0 == g_thread_count) { @@ -135,6 +185,7 @@ static void timer_thread(void *unused) { ct->next = g_completed_threads; g_completed_threads = ct; gpr_mu_unlock(&g_mu); + grpc_exec_ctx_finish(&exec_ctx); gpr_log(GPR_DEBUG, "End timer thread"); } @@ -193,6 +244,8 @@ void grpc_timer_manager_set_threading(bool threaded) { void grpc_kick_poller(void) { gpr_mu_lock(&g_mu); g_kicked = true; + g_has_timed_waiter = false; + ++g_timed_waiter_generation; gpr_cv_signal(&g_cv_wait); gpr_mu_unlock(&g_mu); } From 2e8993f07c90432b847cf615628afadbb5f876b7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 21:25:08 -0700 Subject: [PATCH 127/177] Eliminate some complexity --- src/core/lib/iomgr/timer_manager.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 8b83cc0106c..dd7120695ca 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -44,19 +44,27 @@ typedef struct completed_thread { struct completed_thread *next; } completed_thread; +// global mutex static gpr_mu g_mu; +// are we multi-threaded static bool g_threaded; +// cv to wait until a thread is needed static gpr_cv g_cv_wait; +// cv for notification when threading ends static gpr_cv g_cv_shutdown; +// number of threads in the system static int g_thread_count; +// number of threads sitting around waiting static int g_waiter_count; +// linked list of threads that have completed (and need joining) static completed_thread *g_completed_threads; +// was the manager kicked by the timer system static bool g_kicked; +// is there a thread waiting until the next timer should fire? static bool g_has_timed_waiter; +// generation counter to track which thread is waiting for the next timer static uint64_t g_timed_waiter_generation; -#define MAX_WAITERS 2 - static void timer_thread(void *unused); static void gc_completed_threads(void) { @@ -147,28 +155,21 @@ static void timer_thread(void *unused) { next = inf_future; gpr_log(GPR_DEBUG, "sleep until kicked"); } - bool timed_out = gpr_cv_wait(&g_cv_wait, &g_mu, next); - // if we timed out and we have too many waiters, maybe exit this thread - bool should_stop = (timed_out && g_waiter_count > MAX_WAITERS); - gpr_log(GPR_DEBUG, "wait ended: was_timed:%d timed_out:%d kicked:%d", + gpr_cv_wait(&g_cv_wait, &g_mu, next); + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", my_timed_waiter_generation == g_timed_waiter_generation, - timed_out, g_kicked); + g_kicked); // if this was the timed waiter, then we need to check timers, and flag // that there's now no timed waiter... we'll look for a replacement if // there's work to do after checking timers (code above) if (my_timed_waiter_generation == g_timed_waiter_generation) { g_has_timed_waiter = false; - should_stop = false; } // if this was a kick from the timer system, consume it (and don't stop // this thread yet) if (g_kicked) { grpc_timer_consume_kick(); g_kicked = false; - should_stop = false; - } - if (should_stop) { - break; } } gpr_mu_unlock(&g_mu); From 66918a6106476cbd60137c725529cb22ab958b14 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 4 May 2017 21:27:01 -0700 Subject: [PATCH 128/177] simplify --- src/core/lib/iomgr/timer_manager.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index dd7120695ca..464907b4546 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -116,8 +116,7 @@ static void timer_thread(void *unused) { gpr_mu_lock(&g_mu); // remove a waiter from the pool, and start another thread if necessary --g_waiter_count; - bool start_thread = g_waiter_count == 0; - if (start_thread && g_threaded) { + if (g_waiter_count == 0 && g_threaded) { start_timer_thread_and_unlock(); } else { // if there's no thread waiting with a timeout, kick an existing waiter From cfaa046a0127f9e40e63eaf5490164794d6ab253 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 15:27:40 +0000 Subject: [PATCH 129/177] Integrate new tracers --- src/core/lib/iomgr/ev_epoll1_linux.c | 14 ----- src/core/lib/iomgr/ev_epollex_linux.c | 73 +++++++------------------- src/core/lib/iomgr/ev_epollsig_linux.c | 3 +- src/core/lib/iomgr/ev_posix.c | 2 +- src/core/lib/iomgr/ev_posix.h | 3 +- 5 files changed, 23 insertions(+), 72 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 90c55465558..8aa69cd73a0 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -63,12 +63,8 @@ #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ static grpc_wakeup_fd global_wakeup_fd; static int g_epfd; -static gpr_atm g_timer_kick; /******************************************************************************* * Fd Declarations @@ -512,9 +508,6 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; if (data_ptr == &global_wakeup_fd) { - if (gpr_atm_no_barrier_cas(&g_timer_kick, 1, 0)) { - grpc_timer_consume_kick(); - } gpr_mu_lock(&g_wq_mu); grpc_closure_list_move(&g_wq_items, &exec_ctx->closure_list); gpr_mu_unlock(&g_wq_mu); @@ -799,11 +792,6 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_fd *fd) {} -static grpc_error *kick_poller(void) { - gpr_atm_no_barrier_store(&g_timer_kick, 1); - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - /******************************************************************************* * Workqueue Definitions */ @@ -951,8 +939,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index b9f60d8e3ea..5a7c0448b65 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -64,11 +64,6 @@ #include "src/core/lib/support/block_annotate.h" #include "src/core/lib/support/spinlock.h" -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - /******************************************************************************* * Pollset-set sibling link */ @@ -560,16 +555,6 @@ static grpc_error *pollable_materialize(pollable *p) { int new_epfd = epoll_create1(EPOLL_CLOEXEC); if (new_epfd < 0) { return GRPC_OS_ERROR(errno, "epoll_create1"); - } else { - struct epoll_event ev = { - .events = (uint32_t)(EPOLLIN | EPOLLET | EPOLLEXCLUSIVE), - .data.ptr = &global_wakeup_fd}; - if (epoll_ctl(new_epfd, EPOLL_CTL_ADD, global_wakeup_fd.read_fd, &ev) != - 0) { - grpc_error *err = GRPC_OS_ERROR(errno, "epoll_ctl"); - close(new_epfd); - return err; - } } grpc_error *err = grpc_wakeup_fd_init(&p->wakeup); if (err != GRPC_ERROR_NONE) { @@ -639,22 +624,16 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); -static bool global_wakeup_fd_initialized = false; /* Global state management */ static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - grpc_error *error = GRPC_ERROR_NONE; - static const char *err_desc = "pollset_global_init"; - global_wakeup_fd_initialized = - append_error(&error, grpc_wakeup_fd_init(&global_wakeup_fd), err_desc); pollable_init(&g_empty_pollable, PO_EMPTY_POLLABLE); - return error; + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - if (global_wakeup_fd_initialized) grpc_wakeup_fd_destroy(&global_wakeup_fd); pollable_destroy(&g_empty_pollable); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); @@ -687,7 +666,7 @@ static grpc_error *pollset_kick_all(grpc_pollset *pollset) { static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, grpc_pollset_worker *specific_worker) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kick %p tls_pollset=%p tls_worker=%p " "root_worker=(pollset:%p pollable:%p)", @@ -698,13 +677,13 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, if (specific_worker == NULL) { if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { if (pollset->root_worker == NULL) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_without_poller", p); } pollset->kicked_without_poller = true; return GRPC_ERROR_NONE; } else { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_via_wakeup_fd", p); } grpc_error *err = pollable_materialize(p); @@ -712,25 +691,25 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, return grpc_wakeup_fd_wakeup(&p->wakeup); } } else { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_any_but_awake", p); } return GRPC_ERROR_NONE; } } else if (specific_worker->kicked) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_already_kicked", p); } return GRPC_ERROR_NONE; } else if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_but_awake", p); } specific_worker->kicked = true; return GRPC_ERROR_NONE; } else if (specific_worker == p->root_worker) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_wakeup_fd", p); } grpc_error *err = pollable_materialize(p); @@ -738,7 +717,7 @@ static grpc_error *pollset_kick_inner(grpc_pollset *pollset, pollable *p, specific_worker->kicked = true; return grpc_wakeup_fd_wakeup(&p->wakeup); } else { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p kicked_specific_via_cv", p); } specific_worker->kicked = true; @@ -761,10 +740,6 @@ static grpc_error *pollset_kick(grpc_pollset *pollset, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollable_init(&pollset->pollable, PO_POLLSET); pollset->current_pollable = &g_empty_pollable; @@ -865,7 +840,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, int timeout = poll_deadline_to_millis_timeout(deadline, now); - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p for %dms", pollset, p, timeout); } @@ -882,23 +857,15 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, if (r < 0) return GRPC_OS_ERROR(errno, "epoll_wait"); - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p got %d events", pollset, p, r); } grpc_error *error = GRPC_ERROR_NONE; for (int i = 0; i < r; i++) { void *data_ptr = events[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - if (grpc_polling_trace) { - gpr_log(GPR_DEBUG, "PS:%p poll %p got global_wakeup_fd", pollset, p); - } - - grpc_timer_consume_kick(); - append_error(&error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &p->wakeup) { - if (grpc_polling_trace) { + if (data_ptr == &p->wakeup) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p got pollset_wakeup", pollset, p); } append_error(&error, grpc_wakeup_fd_consume_wakeup(&p->wakeup), err_desc); @@ -908,7 +875,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, bool cancel = (events[i].events & (EPOLLERR | EPOLLHUP)) != 0; bool read_ev = (events[i].events & (EPOLLIN | EPOLLPRI)) != 0; bool write_ev = (events[i].events & EPOLLOUT) != 0; - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p poll %p got fd %p: is_wq=%d cancel=%d read=%d " "write=%d", @@ -994,25 +961,25 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (worker->pollable != &pollset->pollable) { gpr_mu_unlock(&pollset->pollable.po.mu); } - if (grpc_polling_trace && worker->pollable->root_worker != worker) { + if (GRPC_TRACER_ON(grpc_polling_trace) && worker->pollable->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, worker->pollable, worker, poll_deadline_to_millis_timeout(deadline, *now)); } while (do_poll && worker->pollable->root_worker != worker) { if (gpr_cv_wait(&worker->cv, &worker->pollable->po.mu, deadline)) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p timeout_wait %p w=%p", pollset, worker->pollable, worker); } do_poll = false; } else if (worker->kicked) { - if (grpc_polling_trace) { + if (GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p wakeup %p w=%p", pollset, worker->pollable, worker); } do_poll = false; - } else if (grpc_polling_trace && + } else if (GRPC_TRACER_ON(grpc_polling_trace) && worker->pollable->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p spurious_wakeup %p w=%p", pollset, worker->pollable, worker); @@ -1056,7 +1023,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - if (0 && grpc_polling_trace) { + if (0 && GRPC_TRACER_ON(grpc_polling_trace)) { gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 ".%09d deadline=%" PRId64 ".%09d kwp=%d root_worker=%p", pollset, worker_hdl, &worker, now.tv_sec, now.tv_nsec, @@ -1484,8 +1451,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 11be0e70a4e..d9ba77c6f04 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -65,9 +65,8 @@ #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) -/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ #define GRPC_POLLING_TRACE(fmt, ...) \ - if (grpc_polling_trace) { \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index d3302f4a807..e3d53d6d3d1 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -51,7 +51,7 @@ #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" -int grpc_polling_trace = 0; /* Disabled by default */ +grpc_tracer_flag grpc_polling_trace = GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ /** Default poll() function - a pointer so that it can be overridden by some * tests */ diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index d70ed74ae1c..e013a6c9530 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -41,8 +41,9 @@ #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/debug/trace.h" -extern int grpc_polling_trace; /* Disabled by default */ +extern grpc_tracer_flag grpc_polling_trace; /* Disabled by default */ typedef struct grpc_fd grpc_fd; From 6b82f8c4bfaaab5bb20171eb25ec7d7b0510f5f1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 16:13:28 +0000 Subject: [PATCH 130/177] Fix misplaced unlock, cleanup tracing --- src/core/lib/iomgr/timer_manager.c | 43 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/core/lib/iomgr/timer_manager.c b/src/core/lib/iomgr/timer_manager.c index 464907b4546..24085093e70 100644 --- a/src/core/lib/iomgr/timer_manager.c +++ b/src/core/lib/iomgr/timer_manager.c @@ -37,6 +37,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/timer.h" typedef struct completed_thread { @@ -44,6 +45,8 @@ typedef struct completed_thread { struct completed_thread *next; } completed_thread; +extern grpc_tracer_flag grpc_timer_check_trace; + // global mutex static gpr_mu g_mu; // are we multi-threaded @@ -83,10 +86,13 @@ static void gc_completed_threads(void) { } static void start_timer_thread_and_unlock(void) { + GPR_ASSERT(g_threaded); ++g_waiter_count; ++g_thread_count; gpr_mu_unlock(&g_mu); - gpr_log(GPR_DEBUG, "Spawn timer thread"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "Spawn timer thread"); + } gpr_thd_id thd; gpr_thd_options opt = gpr_thd_options_default(); gpr_thd_options_set_joinable(&opt); @@ -122,7 +128,9 @@ static void timer_thread(void *unused) { // if there's no thread waiting with a timeout, kick an existing waiter // so that the next deadline is not missed if (!g_has_timed_waiter) { - gpr_log(GPR_DEBUG, "kick untimed waiter"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "kick untimed waiter"); + } gpr_cv_signal(&g_cv_wait); } gpr_mu_unlock(&g_mu); @@ -149,15 +157,21 @@ static void timer_thread(void *unused) { // cancel an existing one quickly (and when it actually times out it'll // figure stuff out instead of incurring a wakeup) my_timed_waiter_generation = ++g_timed_waiter_generation; - gpr_log(GPR_DEBUG, "sleep for a while"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "sleep for a while"); + } } else { next = inf_future; - gpr_log(GPR_DEBUG, "sleep until kicked"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "sleep until kicked"); + } } gpr_cv_wait(&g_cv_wait, &g_mu, next); - gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", - my_timed_waiter_generation == g_timed_waiter_generation, - g_kicked); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "wait ended: was_timed:%d kicked:%d", + my_timed_waiter_generation == g_timed_waiter_generation, + g_kicked); + } // if this was the timed waiter, then we need to check timers, and flag // that there's now no timed waiter... we'll look for a replacement if // there's work to do after checking timers (code above) @@ -170,8 +184,8 @@ static void timer_thread(void *unused) { grpc_timer_consume_kick(); g_kicked = false; } + gpr_mu_unlock(&g_mu); } - gpr_mu_unlock(&g_mu); } // terminate the thread: drop the waiter count, thread count, and let whomever // stopped the threading stuff know that we're done @@ -186,7 +200,9 @@ static void timer_thread(void *unused) { g_completed_threads = ct; gpr_mu_unlock(&g_mu); grpc_exec_ctx_finish(&exec_ctx); - gpr_log(GPR_DEBUG, "End timer thread"); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "End timer thread"); + } } static void start_threads(void) { @@ -214,11 +230,20 @@ void grpc_timer_manager_init(void) { static void stop_threads(void) { gpr_mu_lock(&g_mu); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "stop timer threads: threaded=%d", g_threaded); + } if (g_threaded) { g_threaded = false; gpr_cv_broadcast(&g_cv_wait); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count); + } while (g_thread_count > 0) { gpr_cv_wait(&g_cv_shutdown, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + if (GRPC_TRACER_ON(grpc_timer_check_trace)) { + gpr_log(GPR_DEBUG, "num timer threads: %d", g_thread_count); + } gc_completed_threads(); } } From 6c8383af699211963debb76e2f96dee09497c26a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 16:54:42 +0000 Subject: [PATCH 131/177] Fix missing edge, add tracing --- src/core/lib/iomgr/ev_epoll1_linux.c | 1 - src/core/lib/iomgr/ev_epollex_linux.c | 26 +++++++++++++++++++++++--- src/core/lib/iomgr/lockfree_event.c | 16 ++++++++++++++++ src/core/lib/iomgr/tcp_client_posix.c | 4 ++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index 8aa69cd73a0..d31eafc70b1 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -57,7 +57,6 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/lockfree_event.h" -#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index 5a7c0448b65..cb6814e89e9 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -582,6 +582,10 @@ static grpc_error *pollable_add_fd(pollable *p, grpc_fd *fd) { const int epfd = p->epfd; GPR_ASSERT(epfd != -1); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "add fd %p to pollable %p", fd, p); + } + gpr_mu_lock(&fd->orphaned_mu); if (fd->orphaned) { gpr_mu_unlock(&fd->orphaned_mu); @@ -961,7 +965,8 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, if (worker->pollable != &pollset->pollable) { gpr_mu_unlock(&pollset->pollable.po.mu); } - if (GRPC_TRACER_ON(grpc_polling_trace) && worker->pollable->root_worker != worker) { + if (GRPC_TRACER_ON(grpc_polling_trace) && + worker->pollable->root_worker != worker) { gpr_log(GPR_DEBUG, "PS:%p wait %p w=%p for %dms", pollset, worker->pollable, worker, poll_deadline_to_millis_timeout(deadline, *now)); @@ -1080,6 +1085,10 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, static const char *err_desc = "pollset_add_fd"; grpc_error *error = GRPC_ERROR_NONE; if (pollset->current_pollable == &g_empty_pollable) { + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, + "PS:%p add fd %p; transition pollable from empty to fd", pollset, + fd); /* empty pollable --> single fd pollable */ append_error(&error, pollset_kick_all(pollset), err_desc); pollset->current_pollable = &fd->pollable; @@ -1088,10 +1097,17 @@ static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, if (!fd_locked) gpr_mu_unlock(&fd->pollable.po.mu); REF_BY(fd, 2, "pollset_pollable"); } else if (pollset->current_pollable == &pollset->pollable) { + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, "PS:%p add fd %p; already multipolling", pollset, fd); append_error(&error, pollable_add_fd(pollset->current_pollable, fd), err_desc); } else if (pollset->current_pollable != &fd->pollable) { grpc_fd *had_fd = (grpc_fd *)pollset->current_pollable; + if (GRPC_TRACER_ON(grpc_polling_trace)) + gpr_log(GPR_DEBUG, + "PS:%p add fd %p; transition pollable from fd %p to multipoller", + pollset, fd, had_fd); + append_error(&error, pollset_kick_all(pollset), err_desc); pollset->current_pollable = &pollset->pollable; if (append_error(&error, pollable_materialize(&pollset->pollable), err_desc)) { @@ -1458,7 +1474,8 @@ static const grpc_event_engine_vtable vtable = { .shutdown_engine = shutdown_engine, }; -const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requested) { +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested) { if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1483,7 +1500,10 @@ const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requeste #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requested) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/lockfree_event.c b/src/core/lib/iomgr/lockfree_event.c index 17e3bbf7278..898ec1cb1bf 100644 --- a/src/core/lib/iomgr/lockfree_event.c +++ b/src/core/lib/iomgr/lockfree_event.c @@ -35,6 +35,10 @@ #include +#include "src/core/lib/debug/trace.h" + +extern grpc_tracer_flag grpc_polling_trace; + /* 'state' holds the to call when the fd is readable or writable respectively. It can contain one of the following values: CLOSURE_READY : The fd has an I/O event of interest but there is no @@ -93,6 +97,10 @@ void grpc_lfev_notify_on(grpc_exec_ctx *exec_ctx, gpr_atm *state, grpc_closure *closure) { while (true) { gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_notify_on: %p curr=%p closure=%p", state, + (void *)curr, closure); + } switch (curr) { case CLOSURE_NOT_READY: { /* CLOSURE_NOT_READY -> . @@ -155,6 +163,10 @@ bool grpc_lfev_set_shutdown(grpc_exec_ctx *exec_ctx, gpr_atm *state, while (true) { gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_set_shutdown: %p curr=%p err=%s", state, + (void *)curr, grpc_error_string(shutdown_err)); + } switch (curr) { case CLOSURE_READY: case CLOSURE_NOT_READY: @@ -200,6 +212,10 @@ void grpc_lfev_set_ready(grpc_exec_ctx *exec_ctx, gpr_atm *state) { while (true) { gpr_atm curr = gpr_atm_no_barrier_load(state); + if (GRPC_TRACER_ON(grpc_polling_trace)) { + gpr_log(GPR_DEBUG, "lfev_set_ready: %p curr=%p", state, (void *)curr); + } + switch (curr) { case CLOSURE_READY: { /* Already ready. We are done here */ diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index ed3fd94a98d..5c7da999e0e 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -331,8 +331,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx, ac->channel_args = grpc_channel_args_copy(channel_args); if (GRPC_TRACER_ON(grpc_tcp_trace)) { - gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting", - ac->addr_str); + gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting fd %p", + ac->addr_str, fdobj); } gpr_mu_lock(&ac->mu); From 3e04383817b60a3fae9292fa21aaf50a9b0ffb8f Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 5 May 2017 10:11:46 -0700 Subject: [PATCH 132/177] Remove extra log line --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 8679f9c85d7..f46dee8a046 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -1283,7 +1283,6 @@ static void shutdown_poller_threads() { gpr_thd_join(g_poller_threads[i]); } - gpr_log(GPR_ERROR, "epoll set delete called"); GRPC_LOG_IF_ERROR("shutdown_poller_threads", error); gpr_free(g_poller_threads); g_poller_threads = NULL; From 924353a7173ed6184f3e0baef8c62d3ccbfa39f5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 17:36:31 +0000 Subject: [PATCH 133/177] Finish merge, disable new pollers until they stabilize --- CMakeLists.txt | 7 ---- Makefile | 7 ---- binding.gyp | 1 - build.yaml | 1 - config.m4 | 1 - gRPC-Core.podspec | 1 - grpc.gemspec | 1 - package.xml | 1 - src/core/lib/iomgr/ev_epoll1_linux.c | 3 ++ .../lib/iomgr/ev_epoll_thread_pool_linux.c | 41 +++++++------------ .../lib/iomgr/ev_epoll_thread_pool_linux.h | 2 +- src/core/lib/iomgr/ev_epollex_linux.c | 2 + src/core/lib/iomgr/ev_epollsig_linux.c | 3 +- src/python/grpcio/grpc_core_dependencies.py | 1 - tools/doxygen/Doxyfile.c++.internal | 1 - tools/doxygen/Doxyfile.core.internal | 1 - .../generated/sources_and_headers.json | 1 - vsprojects/vcxproj/grpc++/grpc++.vcxproj | 2 - .../vcxproj/grpc++/grpc++.vcxproj.filters | 3 -- .../grpc++_unsecure/grpc++_unsecure.vcxproj | 2 - .../grpc++_unsecure.vcxproj.filters | 3 -- vsprojects/vcxproj/grpc/grpc.vcxproj | 2 - vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 3 -- .../grpc_test_util/grpc_test_util.vcxproj | 2 - .../grpc_test_util.vcxproj.filters | 3 -- .../grpc_unsecure/grpc_unsecure.vcxproj | 2 - .../grpc_unsecure.vcxproj.filters | 3 -- 27 files changed, 22 insertions(+), 78 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 007982644f2..581d96bc06e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -940,7 +940,6 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -1274,7 +1273,6 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -1591,7 +1589,6 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -1853,7 +1850,6 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -2280,7 +2276,6 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -2611,7 +2606,6 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c @@ -3386,7 +3380,6 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c src/core/lib/iomgr/ev_epoll1_linux.c - src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_epoll_thread_pool_linux.c src/core/lib/iomgr/ev_epollex_linux.c src/core/lib/iomgr/ev_epollsig_linux.c diff --git a/Makefile b/Makefile index b731dcb8319..e83b73827b4 100644 --- a/Makefile +++ b/Makefile @@ -2923,7 +2923,6 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -3255,7 +3254,6 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -3571,7 +3569,6 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -3805,7 +3802,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -4209,7 +4205,6 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -4548,7 +4543,6 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ @@ -5313,7 +5307,6 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ diff --git a/binding.gyp b/binding.gyp index 37e41efb5c8..bf3d14135df 100644 --- a/binding.gyp +++ b/binding.gyp @@ -675,7 +675,6 @@ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_epollsig_linux.c', diff --git a/build.yaml b/build.yaml index 3b062048b65..1223e052505 100644 --- a/build.yaml +++ b/build.yaml @@ -311,7 +311,6 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c - src/core/lib/iomgr/ev_epoll1_linux.c - - src/core/lib/iomgr/ev_epoll_linux.c - src/core/lib/iomgr/ev_epoll_thread_pool_linux.c - src/core/lib/iomgr/ev_epollex_linux.c - src/core/lib/iomgr/ev_epollsig_linux.c diff --git a/config.m4 b/config.m4 index 8c201f3bbff..e2ad0e11b75 100644 --- a/config.m4 +++ b/config.m4 @@ -109,7 +109,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/ev_epoll1_linux.c \ - src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epollex_linux.c \ src/core/lib/iomgr/ev_epollsig_linux.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 2bbdd910e39..c81a522dfe6 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -494,7 +494,6 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_epollsig_linux.c', diff --git a/grpc.gemspec b/grpc.gemspec index d8226fda7dc..95174d4faf7 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -410,7 +410,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) s.files += %w( src/core/lib/iomgr/ev_epoll1_linux.c ) - s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_thread_pool_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epollex_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epollsig_linux.c ) diff --git a/package.xml b/package.xml index 2eba84bafb0..3ba1df3155a 100644 --- a/package.xml +++ b/package.xml @@ -419,7 +419,6 @@ - diff --git a/src/core/lib/iomgr/ev_epoll1_linux.c b/src/core/lib/iomgr/ev_epoll1_linux.c index d31eafc70b1..ad69f808cdb 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.c +++ b/src/core/lib/iomgr/ev_epoll1_linux.c @@ -948,6 +948,9 @@ static const grpc_event_engine_vtable vtable = { /* It is possible that GLIBC has epoll but the underlying kernel doesn't. * Create a dummy epoll_fd to make sure epoll support is available */ const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) { + /* TODO(ctiller): temporary, until this stabilizes */ + if (!explicit_request) return NULL; + if (!grpc_has_wakeup_fd()) { return NULL; } diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index 8679f9c85d7..cf20fc33462 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -66,9 +66,8 @@ #include "src/core/lib/support/block_annotate.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -static int grpc_polling_trace = 0; /* Disabled by default */ #define GRPC_POLLING_TRACE(fmt, ...) \ - if (grpc_polling_trace) { \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } @@ -77,13 +76,10 @@ static int grpc_polling_trace = 0; /* Disabled by default */ * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a * case occurs. */ -/* TODO: sreek: Right now, this wakes up all pollers. In future we should make - * sure to wake up one polling thread (which can wake up other threads if - * needed) */ -static grpc_wakeup_fd global_wakeup_fd; - struct epoll_set; +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1) + /******************************************************************************* * Fd Declarations */ @@ -362,7 +358,7 @@ static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); @@ -423,7 +419,6 @@ static epoll_set *epoll_set_create(grpc_error **error) { goto done; } - epoll_set_add_wakeup_fd_locked(eps, &global_wakeup_fd, error); epoll_set_add_wakeup_fd_locked(eps, &eps->workqueue_wakeup_fd, error); done: @@ -703,11 +698,10 @@ static void pollset_worker_init(grpc_pollset_worker *worker) { static grpc_error *pollset_global_init(void) { gpr_tls_init(&g_current_thread_pollset); gpr_tls_init(&g_current_thread_worker); - return grpc_wakeup_fd_init(&global_wakeup_fd); + return GRPC_ERROR_NONE; } static void pollset_global_shutdown(void) { - grpc_wakeup_fd_destroy(&global_wakeup_fd); gpr_tls_destroy(&g_current_thread_pollset); gpr_tls_destroy(&g_current_thread_worker); } @@ -802,10 +796,6 @@ static grpc_error *pollset_kick(grpc_pollset *p, return error; } -static grpc_error *kick_poller(void) { - return grpc_wakeup_fd_wakeup(&global_wakeup_fd); -} - static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { gpr_mu_init(&pollset->mu); *mu = &pollset->mu; @@ -870,7 +860,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, /* pollset_shutdown is guaranteed to be called before pollset_destroy. So other * than destroying the mutexes, there is nothing special that needs to be done * here */ -static void pollset_destroy(grpc_pollset *pollset) { +static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { GPR_ASSERT(!pollset_has_workers(pollset)); gpr_mu_destroy(&pollset->mu); } @@ -944,11 +934,7 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps, for (int i = 0; i < ep_rv; ++i) { void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &eps->workqueue_wakeup_fd) { + if (data_ptr == &eps->workqueue_wakeup_fd) { append_error(error, grpc_wakeup_fd_consume_wakeup(&eps->workqueue_wakeup_fd), err_desc); @@ -1152,8 +1138,6 @@ static const grpc_event_engine_vtable vtable = { .pollset_set_add_fd = pollset_set_add_fd, .pollset_set_del_fd = pollset_set_del_fd, - .kick_poller = kick_poller, - .workqueue_ref = workqueue_ref, .workqueue_unref = workqueue_unref, .workqueue_scheduler = workqueue_scheduler, @@ -1227,11 +1211,12 @@ static void shutdown_epoll_sets() { for (size_t i = 0; i < g_num_eps; i++) { EPS_UNREF(&exec_ctx, g_epoll_sets[i], "shutdown_epoll_sets"); } - grpc_exec_ctx_finish(&exec_ctx); + grpc_exec_ctx_flush(&exec_ctx); gpr_free(g_epoll_sets); g_epoll_sets = NULL; - pollset_destroy(&g_read_notifier); + pollset_destroy(&exec_ctx, &g_read_notifier); + grpc_exec_ctx_finish(&exec_ctx); } static void poller_thread_loop(void *arg) { @@ -1306,7 +1291,9 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly) { + if (!requested_explicitly) return NULL; + if (!grpc_has_wakeup_fd()) { return NULL; } @@ -1341,6 +1328,6 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void) { #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll_linux(bool requested_explicitly) { return NULL; } #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h index f4959e3fee7..b94dbe7d5d1 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -37,6 +37,6 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(void); +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.c b/src/core/lib/iomgr/ev_epollex_linux.c index cb6814e89e9..7cb6085e255 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.c +++ b/src/core/lib/iomgr/ev_epollex_linux.c @@ -1476,6 +1476,8 @@ static const grpc_event_engine_vtable vtable = { const grpc_event_engine_vtable *grpc_init_epollex_linux( bool explicitly_requested) { + if (!explicitly_requested) return NULL; + if (!grpc_has_wakeup_fd()) { return NULL; } diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index d9ba77c6f04..32096ed55ff 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -1920,7 +1920,8 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux( } if (!is_grpc_wakeup_signal_initialized) { - if (explicit_request) { + /* TODO(ctiller): when other epoll engines are ready, remove the true || to force this to be explitly chosen if needed */ + if (true || explicit_request) { grpc_use_signal(SIGRTMIN + 6); } else { return NULL; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index c7c341fee78..7bf3381f600 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -98,7 +98,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', 'src/core/lib/iomgr/ev_epoll1_linux.c', - 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_epoll_thread_pool_linux.c', 'src/core/lib/iomgr/ev_epollex_linux.c', 'src/core/lib/iomgr/ev_epollsig_linux.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index e56d36dc536..7455d0c0d23 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -941,7 +941,6 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll1_linux.c \ src/core/lib/iomgr/ev_epoll1_linux.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_epollex_linux.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 8b2e6aea75a..96726a53570 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1080,7 +1080,6 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll1_linux.c \ src/core/lib/iomgr/ev_epoll1_linux.h \ -src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.c \ src/core/lib/iomgr/ev_epoll_thread_pool_linux.h \ src/core/lib/iomgr/ev_epollex_linux.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index ece8695f867..4d434e7f6bc 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7914,7 +7914,6 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.c", "src/core/lib/iomgr/ev_epoll1_linux.h", - "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h", "src/core/lib/iomgr/ev_epollex_linux.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 88250ff3ae3..f71af040926 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -618,8 +618,6 @@ - - diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index d9fa3bf741d..5ddcb0924b7 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -187,9 +187,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 3750bb83f71..6ab9653178b 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -602,8 +602,6 @@ - - diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index d628067bd51..3fbd96324bf 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -172,9 +172,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 89e5522b1bc..b9ec423444a 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -559,8 +559,6 @@ - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 6f5add03a28..9568c8db904 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -67,9 +67,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 5b0a632949d..45ef342c937 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -391,8 +391,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index b96e653a577..f3d9d069a3a 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -124,9 +124,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 34f2493a6ee..ba6b989b55b 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -526,8 +526,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 76ba99fdf7d..620720d05af 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -70,9 +70,6 @@ src\core\lib\iomgr - - src\core\lib\iomgr - src\core\lib\iomgr From bc0ab08652fb93bb030aa97955bd8c109a3d3215 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 10:42:44 -0700 Subject: [PATCH 134/177] clang-format --- .../ext/transport/chttp2/transport/internal.h | 2 +- .../lib/iomgr/ev_epoll_thread_pool_linux.c | 20 +++++++++++-------- .../lib/iomgr/ev_epoll_thread_pool_linux.h | 3 ++- src/core/lib/iomgr/ev_epollex_linux.h | 3 ++- src/core/lib/iomgr/ev_epollsig_linux.c | 9 +++++---- src/core/lib/iomgr/ev_posix.c | 7 ++++--- src/core/lib/iomgr/ev_posix.h | 2 +- src/core/lib/iomgr/iomgr.c | 4 +--- src/core/lib/surface/api_trace.c | 2 +- 9 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index bb5ce60872e..b9d06a84d34 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -671,7 +671,7 @@ typedef enum { #define GRPC_CHTTP2_FLOW_CREDIT_COMMON(phase, transport, id, dst_context, \ dst_var, amount) \ do { \ - if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ + if (GRPC_TRACER_ON(grpc_flowctl_trace)) { \ grpc_chttp2_flowctl_trace(__FILE__, __LINE__, phase, \ GRPC_CHTTP2_FLOWCTL_CREDIT, #dst_context, \ #dst_var, NULL, #amount, transport->is_client, \ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index cf20fc33462..a3a9e76ed40 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -66,9 +66,9 @@ #include "src/core/lib/support/block_annotate.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ -#define GRPC_POLLING_TRACE(fmt, ...) \ - if (GRPC_TRACER_ON(grpc_polling_trace)) { \ - gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } /* The alarm system needs to be able to wakeup 'some poller' sometimes @@ -78,7 +78,7 @@ struct epoll_set; -#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker*)1) +#define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) /******************************************************************************* * Fd Declarations @@ -358,8 +358,8 @@ static void epoll_set_add_wakeup_fd_locked(epoll_set *eps, gpr_asprintf(&err_msg, "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " "error: %d (%s)", - eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), - errno, strerror(errno)); + eps->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), errno, + strerror(errno)); append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); gpr_free(err_msg); } @@ -1291,7 +1291,8 @@ static bool is_epoll_available() { return true; } -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly) { +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly) { if (!requested_explicitly) return NULL; if (!grpc_has_wakeup_fd()) { @@ -1328,6 +1329,9 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux(bool requested_explicitly) { return NULL; } +const grpc_event_engine_vtable *grpc_init_epoll_linux( + bool requested_explicitly) { + return NULL; +} #endif /* defined(GRPC_POSIX_SOCKET) */ #endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h index b94dbe7d5d1..9af776a52e9 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux(bool requested_explicitly); +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( + bool requested_explicitly); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_THREAD_POOL_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollex_linux.h b/src/core/lib/iomgr/ev_epollex_linux.h index a865c491168..a078a7f19a8 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.h +++ b/src/core/lib/iomgr/ev_epollex_linux.h @@ -37,6 +37,7 @@ #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/port.h" -const grpc_event_engine_vtable *grpc_init_epollex_linux(bool explicitly_requested); +const grpc_event_engine_vtable *grpc_init_epollex_linux( + bool explicitly_requested); #endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLLEX_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epollsig_linux.c b/src/core/lib/iomgr/ev_epollsig_linux.c index 32096ed55ff..52362a62f47 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.c +++ b/src/core/lib/iomgr/ev_epollsig_linux.c @@ -65,9 +65,9 @@ #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1) -#define GRPC_POLLING_TRACE(fmt, ...) \ - if (GRPC_TRACER_ON(grpc_polling_trace)) { \ - gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (GRPC_TRACER_ON(grpc_polling_trace)) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ } /* Uncomment the following to enable extra checks on poll_object operations */ @@ -1920,7 +1920,8 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux( } if (!is_grpc_wakeup_signal_initialized) { - /* TODO(ctiller): when other epoll engines are ready, remove the true || to force this to be explitly chosen if needed */ + /* TODO(ctiller): when other epoll engines are ready, remove the true || to + * force this to be explitly chosen if needed */ if (true || explicit_request) { grpc_use_signal(SIGRTMIN + 6); } else { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 0b0535c8523..02c6c746ac2 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -45,14 +45,15 @@ #include #include "src/core/lib/debug/trace.h" -#include "src/core/lib/iomgr/ev_epollex_linux.h" #include "src/core/lib/iomgr/ev_epoll1_linux.h" -#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_epoll_thread_pool_linux.h" +#include "src/core/lib/iomgr/ev_epollex_linux.h" +#include "src/core/lib/iomgr/ev_epollsig_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" -grpc_tracer_flag grpc_polling_trace = GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ +grpc_tracer_flag grpc_polling_trace = + GRPC_TRACER_INITIALIZER(false); /* Disabled by default */ /** Default poll() function - a pointer so that it can be overridden by some * tests */ diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index e013a6c9530..80619aab5fc 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -36,12 +36,12 @@ #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" #include "src/core/lib/iomgr/workqueue.h" -#include "src/core/lib/debug/trace.h" extern grpc_tracer_flag grpc_polling_trace; /* Disabled by default */ diff --git a/src/core/lib/iomgr/iomgr.c b/src/core/lib/iomgr/iomgr.c index 37804d86009..1fd41c2f880 100644 --- a/src/core/lib/iomgr/iomgr.c +++ b/src/core/lib/iomgr/iomgr.c @@ -68,9 +68,7 @@ void grpc_iomgr_init(void) { grpc_iomgr_platform_init(); } -void grpc_iomgr_start(void) { - grpc_timer_manager_init(); -} +void grpc_iomgr_start(void) { grpc_timer_manager_init(); } static size_t count_objects(void) { grpc_iomgr_object *obj; diff --git a/src/core/lib/surface/api_trace.c b/src/core/lib/surface/api_trace.c index 55c69cd5b0b..d8941cdf426 100644 --- a/src/core/lib/surface/api_trace.c +++ b/src/core/lib/surface/api_trace.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/debug/trace.h" #include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/debug/trace.h" grpc_tracer_flag grpc_api_trace = GRPC_TRACER_INITIALIZER(false); From 6e0df1478ebfa37c3b5f198b41c119b4e9e3e464 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 10:42:57 -0700 Subject: [PATCH 135/177] Header formatting --- src/core/lib/iomgr/timer_manager.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/timer_manager.h b/src/core/lib/iomgr/timer_manager.h index 0b21262b1a2..46729ccea62 100644 --- a/src/core/lib/iomgr/timer_manager.h +++ b/src/core/lib/iomgr/timer_manager.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_CORE_IOMGR_TIMER_MANAGER_H -#define GRPC_CORE_IOMGR_TIMER_MANAGER_H +#ifndef GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H +#define GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H #include @@ -49,4 +49,4 @@ void grpc_timer_manager_set_threading(bool enabled); * disabled */ void grpc_timer_manager_tick(void); -#endif +#endif /* GRPC_CORE_LIB_IOMGR_TIMER_MANAGER_H */ From 1f47730377077d37534d5d8215c617197e7cb368 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 May 2017 11:01:25 -0700 Subject: [PATCH 136/177] Fix deps between grpc and tracer system --- BUILD | 11 ++++++-- CMakeLists.txt | 14 +++++----- Makefile | 14 +++++----- binding.gyp | 2 +- build.yaml | 12 +++++++-- config.m4 | 2 +- gRPC-Core.podspec | 6 ++--- grpc.gemspec | 4 +-- package.xml | 4 +-- src/python/grpcio/grpc_core_dependencies.py | 2 +- .../generated/sources_and_headers.json | 26 +++++++++++++++---- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 6 ++--- .../vcxproj/grpc++/grpc++.vcxproj.filters | 12 ++++----- .../grpc++_unsecure/grpc++_unsecure.vcxproj | 6 ++--- .../grpc++_unsecure.vcxproj.filters | 12 ++++----- vsprojects/vcxproj/grpc/grpc.vcxproj | 6 ++--- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 12 ++++----- .../grpc_test_util/grpc_test_util.vcxproj | 6 ++--- .../grpc_test_util.vcxproj.filters | 12 ++++----- .../grpc_unsecure/grpc_unsecure.vcxproj | 6 ++--- .../grpc_unsecure.vcxproj.filters | 12 ++++----- 21 files changed, 109 insertions(+), 78 deletions(-) diff --git a/BUILD b/BUILD index 1195747c32a..53ce78610d5 100644 --- a/BUILD +++ b/BUILD @@ -449,6 +449,13 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_trace", + srcs = ["src/core/lib/debug/trace.c"], + hdrs = ["src/core/lib/debug/trace.h"], + deps = [":gpr"], +) + grpc_cc_library( name = "grpc_base", srcs = [ @@ -461,7 +468,6 @@ grpc_cc_library( "src/core/lib/channel/handshaker_registry.c", "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", - "src/core/lib/debug/trace.c", "src/core/lib/http/format_request.c", "src/core/lib/http/httpcli.c", "src/core/lib/http/parser.c", @@ -588,7 +594,6 @@ grpc_cc_library( "src/core/lib/channel/handshaker_registry.h", "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", - "src/core/lib/debug/trace.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", @@ -709,6 +714,7 @@ grpc_cc_library( deps = [ "gpr_base", "grpc_codegen", + "grpc_trace", ], ) @@ -1246,6 +1252,7 @@ grpc_cc_library( language = "c", deps = [ "gpr", + "grpc_trace", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 581d96bc06e..2ef31917d75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -928,7 +928,6 @@ add_library(grpc src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1044,6 +1043,7 @@ add_library(grpc src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c src/core/ext/transport/chttp2/transport/bin_decoder.c src/core/ext/transport/chttp2/transport/bin_encoder.c @@ -1261,7 +1261,6 @@ add_library(grpc_cronet src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1377,6 +1376,7 @@ add_library(grpc_cronet src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c src/core/ext/transport/cronet/client/secure/cronet_channel_create.c src/core/ext/transport/cronet/transport/cronet_api_dummy.c src/core/ext/transport/cronet/transport/cronet_transport.c @@ -1577,7 +1577,6 @@ add_library(grpc_test_util src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1693,6 +1692,7 @@ add_library(grpc_test_util src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c ) if(WIN32 AND MSVC) @@ -1838,7 +1838,6 @@ add_library(grpc_unsecure src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1954,6 +1953,7 @@ add_library(grpc_unsecure src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c src/core/ext/transport/chttp2/server/insecure/server_chttp2.c src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c src/core/ext/transport/chttp2/transport/bin_decoder.c @@ -2264,7 +2264,6 @@ add_library(grpc++ src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -2380,6 +2379,7 @@ add_library(grpc++ src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c third_party/nanopb/pb_common.c third_party/nanopb/pb_decode.c third_party/nanopb/pb_encode.c @@ -2594,7 +2594,6 @@ add_library(grpc++_cronet src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -2710,6 +2709,7 @@ add_library(grpc++_cronet src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c third_party/nanopb/pb_common.c third_party/nanopb/pb_decode.c third_party/nanopb/pb_encode.c @@ -3368,7 +3368,6 @@ add_library(grpc++_unsecure src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c - src/core/lib/debug/trace.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -3484,6 +3483,7 @@ add_library(grpc++_unsecure src/core/lib/transport/timeout_encoding.c src/core/lib/transport/transport.c src/core/lib/transport/transport_op_string.c + src/core/lib/debug/trace.c third_party/nanopb/pb_common.c third_party/nanopb/pb_decode.c third_party/nanopb/pb_encode.c diff --git a/Makefile b/Makefile index e83b73827b4..60759645ec8 100644 --- a/Makefile +++ b/Makefile @@ -2911,7 +2911,6 @@ LIBGRPC_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3027,6 +3026,7 @@ LIBGRPC_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c \ src/core/ext/transport/chttp2/transport/bin_decoder.c \ src/core/ext/transport/chttp2/transport/bin_encoder.c \ @@ -3242,7 +3242,6 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3358,6 +3357,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ src/core/ext/transport/cronet/client/secure/cronet_channel_create.c \ src/core/ext/transport/cronet/transport/cronet_api_dummy.c \ src/core/ext/transport/cronet/transport/cronet_transport.c \ @@ -3557,7 +3557,6 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3673,6 +3672,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ PUBLIC_HEADERS_C += \ include/grpc/byte_buffer.h \ @@ -3790,7 +3790,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3906,6 +3905,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c \ src/core/ext/transport/chttp2/transport/bin_decoder.c \ @@ -4193,7 +4193,6 @@ LIBGRPC++_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -4309,6 +4308,7 @@ LIBGRPC++_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ @@ -4531,7 +4531,6 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -4647,6 +4646,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ @@ -5295,7 +5295,6 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -5411,6 +5410,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_encode.c \ diff --git a/binding.gyp b/binding.gyp index bf3d14135df..3f4e002febf 100644 --- a/binding.gyp +++ b/binding.gyp @@ -663,7 +663,6 @@ 'src/core/lib/channel/handshaker_registry.c', 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', - 'src/core/lib/debug/trace.c', 'src/core/lib/http/format_request.c', 'src/core/lib/http/httpcli.c', 'src/core/lib/http/parser.c', @@ -779,6 +778,7 @@ 'src/core/lib/transport/timeout_encoding.c', 'src/core/lib/transport/transport.c', 'src/core/lib/transport/transport_op_string.c', + 'src/core/lib/debug/trace.c', 'src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c', 'src/core/ext/transport/chttp2/transport/bin_decoder.c', 'src/core/ext/transport/chttp2/transport/bin_encoder.c', diff --git a/build.yaml b/build.yaml index 1223e052505..e636ab17588 100644 --- a/build.yaml +++ b/build.yaml @@ -187,7 +187,6 @@ filegroups: - src/core/lib/channel/handshaker_registry.h - src/core/lib/compression/algorithm_metadata.h - src/core/lib/compression/message_compress.h - - src/core/lib/debug/trace.h - src/core/lib/http/format_request.h - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h @@ -299,7 +298,6 @@ filegroups: - src/core/lib/channel/handshaker_registry.c - src/core/lib/compression/compression.c - src/core/lib/compression/message_compress.c - - src/core/lib/debug/trace.c - src/core/lib/http/format_request.c - src/core/lib/http/httpcli.c - src/core/lib/http/parser.c @@ -419,6 +417,7 @@ filegroups: - gpr uses: - grpc_codegen + - grpc_trace - name: grpc_client_channel headers: - src/core/ext/filters/client_channel/client_channel.h @@ -694,6 +693,13 @@ filegroups: deps: - grpc - gpr_test_util +- name: grpc_trace + headers: + - src/core/lib/debug/trace.h + src: + - src/core/lib/debug/trace.c + deps: + - gpr - name: grpc_transport_chttp2 headers: - src/core/ext/transport/chttp2/transport/bin_decoder.h @@ -842,6 +848,8 @@ filegroups: deps: - gpr secure: true + uses: + - grpc_trace - name: grpc++_base language: c++ public_headers: diff --git a/config.m4 b/config.m4 index e2ad0e11b75..a2a651aac87 100644 --- a/config.m4 +++ b/config.m4 @@ -97,7 +97,6 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ - src/core/lib/debug/trace.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -213,6 +212,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/transport/timeout_encoding.c \ src/core/lib/transport/transport.c \ src/core/lib/transport/transport_op_string.c \ + src/core/lib/debug/trace.c \ src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c \ src/core/ext/transport/chttp2/transport/bin_decoder.c \ src/core/ext/transport/chttp2/transport/bin_encoder.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c81a522dfe6..1867a813c42 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -269,7 +269,6 @@ Pod::Spec.new do |s| 'src/core/lib/channel/handshaker_registry.h', 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', - 'src/core/lib/debug/trace.h', 'src/core/lib/http/format_request.h', 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', @@ -371,6 +370,7 @@ Pod::Spec.new do |s| 'src/core/lib/transport/timeout_encoding.h', 'src/core/lib/transport/transport.h', 'src/core/lib/transport/transport_impl.h', + 'src/core/lib/debug/trace.h', 'src/core/ext/transport/chttp2/transport/bin_decoder.h', 'src/core/ext/transport/chttp2/transport/bin_encoder.h', 'src/core/ext/transport/chttp2/transport/chttp2_transport.h', @@ -482,7 +482,6 @@ Pod::Spec.new do |s| 'src/core/lib/channel/handshaker_registry.c', 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', - 'src/core/lib/debug/trace.c', 'src/core/lib/http/format_request.c', 'src/core/lib/http/httpcli.c', 'src/core/lib/http/parser.c', @@ -598,6 +597,7 @@ Pod::Spec.new do |s| 'src/core/lib/transport/timeout_encoding.c', 'src/core/lib/transport/transport.c', 'src/core/lib/transport/transport_op_string.c', + 'src/core/lib/debug/trace.c', 'src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c', 'src/core/ext/transport/chttp2/transport/bin_decoder.c', 'src/core/ext/transport/chttp2/transport/bin_encoder.c', @@ -745,7 +745,6 @@ Pod::Spec.new do |s| 'src/core/lib/channel/handshaker_registry.h', 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', - 'src/core/lib/debug/trace.h', 'src/core/lib/http/format_request.h', 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', @@ -847,6 +846,7 @@ Pod::Spec.new do |s| 'src/core/lib/transport/timeout_encoding.h', 'src/core/lib/transport/transport.h', 'src/core/lib/transport/transport_impl.h', + 'src/core/lib/debug/trace.h', 'src/core/ext/transport/chttp2/transport/bin_decoder.h', 'src/core/ext/transport/chttp2/transport/bin_encoder.h', 'src/core/ext/transport/chttp2/transport/chttp2_transport.h', diff --git a/grpc.gemspec b/grpc.gemspec index 95174d4faf7..8aaf00d3ecb 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -185,7 +185,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/channel/handshaker_registry.h ) s.files += %w( src/core/lib/compression/algorithm_metadata.h ) s.files += %w( src/core/lib/compression/message_compress.h ) - s.files += %w( src/core/lib/debug/trace.h ) s.files += %w( src/core/lib/http/format_request.h ) s.files += %w( src/core/lib/http/httpcli.h ) s.files += %w( src/core/lib/http/parser.h ) @@ -287,6 +286,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/transport/timeout_encoding.h ) s.files += %w( src/core/lib/transport/transport.h ) s.files += %w( src/core/lib/transport/transport_impl.h ) + s.files += %w( src/core/lib/debug/trace.h ) s.files += %w( src/core/ext/transport/chttp2/transport/bin_decoder.h ) s.files += %w( src/core/ext/transport/chttp2/transport/bin_encoder.h ) s.files += %w( src/core/ext/transport/chttp2/transport/chttp2_transport.h ) @@ -398,7 +398,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/channel/handshaker_registry.c ) s.files += %w( src/core/lib/compression/compression.c ) s.files += %w( src/core/lib/compression/message_compress.c ) - s.files += %w( src/core/lib/debug/trace.c ) s.files += %w( src/core/lib/http/format_request.c ) s.files += %w( src/core/lib/http/httpcli.c ) s.files += %w( src/core/lib/http/parser.c ) @@ -514,6 +513,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/transport/timeout_encoding.c ) s.files += %w( src/core/lib/transport/transport.c ) s.files += %w( src/core/lib/transport/transport_op_string.c ) + s.files += %w( src/core/lib/debug/trace.c ) s.files += %w( src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c ) s.files += %w( src/core/ext/transport/chttp2/transport/bin_decoder.c ) s.files += %w( src/core/ext/transport/chttp2/transport/bin_encoder.c ) diff --git a/package.xml b/package.xml index 3ba1df3155a..46617a47d5c 100644 --- a/package.xml +++ b/package.xml @@ -194,7 +194,6 @@ - @@ -296,6 +295,7 @@ + @@ -407,7 +407,6 @@ - @@ -523,6 +522,7 @@ + diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 7bf3381f600..3eccc9e917f 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -86,7 +86,6 @@ CORE_SOURCE_FILES = [ 'src/core/lib/channel/handshaker_registry.c', 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', - 'src/core/lib/debug/trace.c', 'src/core/lib/http/format_request.c', 'src/core/lib/http/httpcli.c', 'src/core/lib/http/parser.c', @@ -202,6 +201,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/transport/timeout_encoding.c', 'src/core/lib/transport/transport.c', 'src/core/lib/transport/transport_op_string.c', + 'src/core/lib/debug/trace.c', 'src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c', 'src/core/ext/transport/chttp2/transport/bin_decoder.c', 'src/core/ext/transport/chttp2/transport/bin_encoder.c', diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 4d434e7f6bc..74ae6acb8e5 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7732,7 +7732,8 @@ { "deps": [ "gpr", - "grpc_codegen" + "grpc_codegen", + "grpc_trace" ], "headers": [ "include/grpc/byte_buffer.h", @@ -7755,7 +7756,6 @@ "src/core/lib/channel/handshaker_registry.h", "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", - "src/core/lib/debug/trace.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", @@ -7891,8 +7891,6 @@ "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/message_compress.h", - "src/core/lib/debug/trace.c", - "src/core/lib/debug/trace.h", "src/core/lib/http/format_request.c", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.c", @@ -8614,6 +8612,23 @@ "third_party": false, "type": "filegroup" }, + { + "deps": [ + "gpr" + ], + "headers": [ + "src/core/lib/debug/trace.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_trace", + "src": [ + "src/core/lib/debug/trace.c", + "src/core/lib/debug/trace.h" + ], + "third_party": false, + "type": "filegroup" + }, { "deps": [ "gpr", @@ -8865,7 +8880,8 @@ }, { "deps": [ - "gpr" + "gpr", + "grpc_trace" ], "headers": [ "src/core/tsi/fake_transport_security.h", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index f71af040926..3b889047cc9 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -386,7 +386,6 @@ - @@ -488,6 +487,7 @@ + @@ -594,8 +594,6 @@ - - @@ -826,6 +824,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 5ddcb0924b7..8d916898222 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -151,9 +151,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -499,6 +496,9 @@ src\core\lib\transport + + src\core\lib\debug + third_party\nanopb @@ -893,9 +893,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -1199,6 +1196,9 @@ src\core\lib\transport + + src\core\lib\debug + third_party\nanopb diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 6ab9653178b..4cc3e0e3768 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -380,7 +380,6 @@ - @@ -482,6 +481,7 @@ + @@ -578,8 +578,6 @@ - - @@ -810,6 +808,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 3fbd96324bf..85f5c06efc7 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -136,9 +136,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -484,6 +481,9 @@ src\core\lib\transport + + src\core\lib\debug + third_party\nanopb @@ -860,9 +860,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -1166,6 +1163,9 @@ src\core\lib\transport + + src\core\lib\debug + third_party\nanopb diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index b9ec423444a..0df99a2b4a3 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -310,7 +310,6 @@ - @@ -412,6 +411,7 @@ + @@ -535,8 +535,6 @@ - - @@ -767,6 +765,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 9568c8db904..7326825efe5 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -31,9 +31,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -379,6 +376,9 @@ src\core\lib\transport + + src\core\lib\debug + src\core\ext\transport\chttp2\server\secure @@ -860,9 +860,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -1166,6 +1163,9 @@ src\core\lib\transport + + src\core\lib\debug + src\core\ext\transport\chttp2\transport diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 45ef342c937..6bdfad921e4 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -205,7 +205,6 @@ - @@ -307,6 +306,7 @@ + @@ -367,8 +367,6 @@ - - @@ -599,6 +597,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index f3d9d069a3a..9e20d02d4c6 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -88,9 +88,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -436,6 +433,9 @@ src\core\lib\transport + + src\core\lib\debug + @@ -608,9 +608,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -914,6 +911,9 @@ src\core\lib\transport + + src\core\lib\debug + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index ba6b989b55b..015db7ae4d4 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -300,7 +300,6 @@ - @@ -402,6 +401,7 @@ + @@ -502,8 +502,6 @@ - - @@ -734,6 +732,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 620720d05af..bc0aa7822af 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -34,9 +34,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -382,6 +379,9 @@ src\core\lib\transport + + src\core\lib\debug + src\core\ext\transport\chttp2\server\insecure @@ -770,9 +770,6 @@ src\core\lib\compression - - src\core\lib\debug - src\core\lib\http @@ -1076,6 +1073,9 @@ src\core\lib\transport + + src\core\lib\debug + src\core\ext\transport\chttp2\transport From 5a1a3b49f47afeeb1b7eb2e4ea84432cce98c314 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Fri, 5 May 2017 13:14:45 -0700 Subject: [PATCH 137/177] Separate this into a new poller --- CMakeLists.txt | 7 + Makefile | 7 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + .../iomgr/ev_epoll_limited_pollers_linux.c | 2156 +++++++++++++++++ .../iomgr/ev_epoll_limited_pollers_linux.h | 42 + src/core/lib/iomgr/ev_epoll_linux.c | 399 +-- src/core/lib/iomgr/ev_posix.c | 2 + src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 3 + vsprojects/vcxproj/grpc++/grpc++.vcxproj | 3 + .../vcxproj/grpc++/grpc++.vcxproj.filters | 6 + .../grpc++_unsecure/grpc++_unsecure.vcxproj | 3 + .../grpc++_unsecure.vcxproj.filters | 6 + vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../grpc_test_util/grpc_test_util.vcxproj | 3 + .../grpc_test_util.vcxproj.filters | 6 + .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + 26 files changed, 2365 insertions(+), 312 deletions(-) create mode 100644 src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c create mode 100644 src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f6c5f9e56d5..68c58470ebc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -938,6 +938,7 @@ add_library(grpc src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -1266,6 +1267,7 @@ add_library(grpc_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -1577,6 +1579,7 @@ add_library(grpc_test_util src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -1833,6 +1836,7 @@ add_library(grpc_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -2254,6 +2258,7 @@ add_library(grpc++ src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -2579,6 +2584,7 @@ add_library(grpc++_cronet src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c @@ -3348,6 +3354,7 @@ add_library(grpc++_unsecure src/core/lib/iomgr/endpoint_pair_uv.c src/core/lib/iomgr/endpoint_pair_windows.c src/core/lib/iomgr/error.c + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c src/core/lib/iomgr/ev_epoll_linux.c src/core/lib/iomgr/ev_poll_posix.c src/core/lib/iomgr/ev_posix.c diff --git a/Makefile b/Makefile index 1a8ee553ca3..1fdf64d03fd 100644 --- a/Makefile +++ b/Makefile @@ -2921,6 +2921,7 @@ LIBGRPC_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -3247,6 +3248,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -3557,6 +3559,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -3785,6 +3788,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -4183,6 +4187,7 @@ LIBGRPC++_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -4516,6 +4521,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -5275,6 +5281,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ diff --git a/binding.gyp b/binding.gyp index c8dd5c1cfdd..bf7c6a15290 100644 --- a/binding.gyp +++ b/binding.gyp @@ -674,6 +674,7 @@ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c', 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', diff --git a/build.yaml b/build.yaml index b2930602346..2ae2f1b4081 100644 --- a/build.yaml +++ b/build.yaml @@ -197,6 +197,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair.h - src/core/lib/iomgr/error.h - src/core/lib/iomgr/error_internal.h + - src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h - src/core/lib/iomgr/ev_epoll_linux.h - src/core/lib/iomgr/ev_poll_posix.h - src/core/lib/iomgr/ev_posix.h @@ -304,6 +305,7 @@ filegroups: - src/core/lib/iomgr/endpoint_pair_uv.c - src/core/lib/iomgr/endpoint_pair_windows.c - src/core/lib/iomgr/error.c + - src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c - src/core/lib/iomgr/ev_epoll_linux.c - src/core/lib/iomgr/ev_poll_posix.c - src/core/lib/iomgr/ev_posix.c diff --git a/config.m4 b/config.m4 index 1c0c6d92fc4..f54033c0c22 100644 --- a/config.m4 +++ b/config.m4 @@ -108,6 +108,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/iomgr/endpoint_pair_uv.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ + src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 241eba01dd2..cdc5e5837da 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -279,6 +279,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h', 'src/core/lib/iomgr/ev_epoll_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', @@ -487,6 +488,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c', 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', @@ -744,6 +746,7 @@ Pod::Spec.new do |s| 'src/core/lib/iomgr/endpoint_pair.h', 'src/core/lib/iomgr/error.h', 'src/core/lib/iomgr/error_internal.h', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h', 'src/core/lib/iomgr/ev_epoll_linux.h', 'src/core/lib/iomgr/ev_poll_posix.h', 'src/core/lib/iomgr/ev_posix.h', diff --git a/grpc.gemspec b/grpc.gemspec index b96f3cbab54..22151fb29ba 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -195,6 +195,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair.h ) s.files += %w( src/core/lib/iomgr/error.h ) s.files += %w( src/core/lib/iomgr/error_internal.h ) + s.files += %w( src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.h ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.h ) s.files += %w( src/core/lib/iomgr/ev_posix.h ) @@ -403,6 +404,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/iomgr/endpoint_pair_uv.c ) s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c ) s.files += %w( src/core/lib/iomgr/error.c ) + s.files += %w( src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c ) s.files += %w( src/core/lib/iomgr/ev_epoll_linux.c ) s.files += %w( src/core/lib/iomgr/ev_poll_posix.c ) s.files += %w( src/core/lib/iomgr/ev_posix.c ) diff --git a/package.xml b/package.xml index d47708efa4e..15c573bd48a 100644 --- a/package.xml +++ b/package.xml @@ -204,6 +204,7 @@ + @@ -412,6 +413,7 @@ + diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c new file mode 100644 index 00000000000..1b1c597d827 --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c @@ -0,0 +1,2156 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/iomgr/port.h" + +/* This polling engine is only relevant on linux kernels supporting epoll() */ +#ifdef GRPC_LINUX_EPOLL + +#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/lockfree_event.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/iomgr/wakeup_fd_posix.h" +#include "src/core/lib/iomgr/workqueue.h" +#include "src/core/lib/profiling/timers.h" +#include "src/core/lib/support/block_annotate.h" +#include "src/core/lib/support/env.h" + +/* TODO: sreek - Move this to init.c and initialize this like other tracers. */ +static int grpc_polling_trace = 0; /* Disabled by default */ +#define GRPC_POLLING_TRACE(fmt, ...) \ + if (grpc_polling_trace) { \ + gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \ + } + +/* Uncomment the following to enable extra checks on poll_object operations */ +/* #define PO_DEBUG */ + +/* The maximum number of polling threads per polling island. By default no + limit */ +static int g_max_pollers_per_pi = INT_MAX; + +static int grpc_wakeup_signal = -1; +static bool is_grpc_wakeup_signal_initialized = false; + +/* TODO: sreek: Right now, this wakes up all pollers. In future we should make + * sure to wake up one polling thread (which can wake up other threads if + * needed) */ +static grpc_wakeup_fd global_wakeup_fd; + +/* Implements the function defined in grpc_posix.h. This function might be + * called before even calling grpc_init() to set either a different signal to + * use. If signum == -1, then the use of signals is disabled */ +static void grpc_use_signal(int signum) { + grpc_wakeup_signal = signum; + is_grpc_wakeup_signal_initialized = true; + + if (grpc_wakeup_signal < 0) { + gpr_log(GPR_INFO, + "Use of signals is disabled. Epoll engine will not be used"); + } else { + gpr_log(GPR_INFO, "epoll engine will be using signal: %d", + grpc_wakeup_signal); + } +} + +struct polling_island; + +typedef enum { + POLL_OBJ_FD, + POLL_OBJ_POLLSET, + POLL_OBJ_POLLSET_SET +} poll_obj_type; + +typedef struct poll_obj { +#ifdef PO_DEBUG + poll_obj_type obj_type; +#endif + gpr_mu mu; + struct polling_island *pi; +} poll_obj; + +static const char *poll_obj_string(poll_obj_type po_type) { + switch (po_type) { + case POLL_OBJ_FD: + return "fd"; + case POLL_OBJ_POLLSET: + return "pollset"; + case POLL_OBJ_POLLSET_SET: + return "pollset_set"; + } + + GPR_UNREACHABLE_CODE(return "UNKNOWN"); +} + +/******************************************************************************* + * Fd Declarations + */ + +#define FD_FROM_PO(po) ((grpc_fd *)(po)) + +struct grpc_fd { + poll_obj po; + + int fd; + /* refst format: + bit 0 : 1=Active / 0=Orphaned + bits 1-n : refcount + Ref/Unref by two to avoid altering the orphaned bit */ + gpr_atm refst; + + /* The fd is either closed or we relinquished control of it. In either + cases, this indicates that the 'fd' on this structure is no longer + valid */ + bool orphaned; + + gpr_atm read_closure; + gpr_atm write_closure; + + struct grpc_fd *freelist_next; + grpc_closure *on_done_closure; + + /* The pollset that last noticed that the fd is readable. The actual type + * stored in this is (grpc_pollset *) */ + gpr_atm read_notifier_pollset; + + grpc_iomgr_object iomgr_object; +}; + +/* Reference counting for fds */ +// #define GRPC_FD_REF_COUNT_DEBUG +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, int line); +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line); +#define GRPC_FD_REF(fd, reason) fd_ref(fd, reason, __FILE__, __LINE__) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd, reason, __FILE__, __LINE__) +#else +static void fd_ref(grpc_fd *fd); +static void fd_unref(grpc_fd *fd); +#define GRPC_FD_REF(fd, reason) fd_ref(fd) +#define GRPC_FD_UNREF(fd, reason) fd_unref(fd) +#endif + +static void fd_global_init(void); +static void fd_global_shutdown(void); + +/******************************************************************************* + * Polling island Declarations + */ + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG + +#define PI_ADD_REF(p, r) pi_add_ref_dbg((p), (r), __FILE__, __LINE__) +#define PI_UNREF(exec_ctx, p, r) \ + pi_unref_dbg((exec_ctx), (p), (r), __FILE__, __LINE__) + +#else /* defined(GRPC_WORKQUEUE_REFCOUNT_DEBUG) */ + +#define PI_ADD_REF(p, r) pi_add_ref((p)) +#define PI_UNREF(exec_ctx, p, r) pi_unref((exec_ctx), (p)) + +#endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ + +typedef struct worker_node { + struct worker_node *next; + struct worker_node *prev; +} worker_node; + +/* This is also used as grpc_workqueue (by directly casing it) */ +typedef struct polling_island { + grpc_closure_scheduler workqueue_scheduler; + + gpr_mu mu; + /* Ref count. Use PI_ADD_REF() and PI_UNREF() macros to increment/decrement + the refcount. + Once the ref count becomes zero, this structure is destroyed which means + we should ensure that there is never a scenario where a PI_ADD_REF() is + racing with a PI_UNREF() that just made the ref_count zero. */ + gpr_atm ref_count; + + /* Pointer to the polling_island this merged into. + * merged_to value is only set once in polling_island's lifetime (and that too + * only if the island is merged with another island). Because of this, we can + * use gpr_atm type here so that we can do atomic access on this and reduce + * lock contention on 'mu' mutex. + * + * Note that if this field is not NULL (i.e not 0), all the remaining fields + * (except mu and ref_count) are invalid and must be ignored. */ + gpr_atm merged_to; + + /* Number of threads currently polling on this island */ + gpr_atm poller_count; + /* Mutex guarding the read end of the workqueue (must be held to pop from + * workqueue_items) */ + gpr_mu workqueue_read_mu; + /* Queue of closures to be executed */ + gpr_mpscq workqueue_items; + /* Count of items in workqueue_items */ + gpr_atm workqueue_item_count; + /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ + grpc_wakeup_fd workqueue_wakeup_fd; + + /* The list of workers waiting to do polling on this polling island */ + gpr_mu worker_list_mu; + worker_node worker_list_head; + + /* The fd of the underlying epoll set */ + int epoll_fd; + + /* The file descriptors in the epoll set */ + size_t fd_cnt; + size_t fd_capacity; + grpc_fd **fds; +} polling_island; + +/******************************************************************************* + * Pollset Declarations + */ +#define WORKER_FROM_WORKER_LIST_NODE(p) \ + (struct grpc_pollset_worker *)(((char *)(p)) - \ + offsetof(grpc_pollset_worker, pi_list_link)) +struct grpc_pollset_worker { + /* Thread id of this worker */ + pthread_t pt_id; + + /* Used to prevent a worker from getting kicked multiple times */ + gpr_atm is_kicked; + + struct grpc_pollset_worker *next; + struct grpc_pollset_worker *prev; + + /* Indicates if it is this worker's turn to do epoll */ + gpr_atm is_polling_turn; + + /* Node in the polling island's worker list. */ + worker_node pi_list_link; +}; + +struct grpc_pollset { + poll_obj po; + + grpc_pollset_worker root_worker; + bool kicked_without_pollers; + + bool shutting_down; /* Is the pollset shutting down ? */ + bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */ + grpc_closure *shutdown_done; /* Called after after shutdown is complete */ +}; + +/******************************************************************************* + * Pollset-set Declarations + */ +struct grpc_pollset_set { + poll_obj po; +}; + +/******************************************************************************* + * Common helpers + */ + +static bool append_error(grpc_error **composite, grpc_error *error, + const char *desc) { + if (error == GRPC_ERROR_NONE) return true; + if (*composite == GRPC_ERROR_NONE) { + *composite = GRPC_ERROR_CREATE_FROM_COPIED_STRING(desc); + } + *composite = grpc_error_add_child(*composite, error); + return false; +} + +/******************************************************************************* + * Polling island Definitions + */ + +/* The wakeup fd that is used to wake up all threads in a Polling island. This + is useful in the polling island merge operation where we need to wakeup all + the threads currently polling the smaller polling island (so that they can + start polling the new/merged polling island) + + NOTE: This fd is initialized to be readable and MUST NOT be consumed i.e the + threads that woke up MUST NOT call grpc_wakeup_fd_consume_wakeup() */ +static grpc_wakeup_fd polling_island_wakeup_fd; + +/* The polling island being polled right now. + See comments in workqueue_maybe_wakeup for why this is tracked. */ +static __thread polling_island *g_current_thread_polling_island; + +/* Forward declaration */ +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi); +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error); + +#ifdef GRPC_TSAN +/* Currently TSAN may incorrectly flag data races between epoll_ctl and + epoll_wait for any grpc_fd structs that are added to the epoll set via + epoll_ctl and are returned (within a very short window) via epoll_wait(). + + To work-around this race, we establish a happens-before relation between + the code just-before epoll_ctl() and the code after epoll_wait() by using + this atomic */ +gpr_atm g_epoll_sync; +#endif /* defined(GRPC_TSAN) */ + +static const grpc_closure_scheduler_vtable workqueue_scheduler_vtable = { + workqueue_enqueue, workqueue_enqueue, "workqueue"}; + +static void pi_add_ref(polling_island *pi); +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi); + +#ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG +static void pi_add_ref_dbg(polling_island *pi, const char *reason, + const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_add_ref(pi); + gpr_log(GPR_DEBUG, "Add ref pi: %p, old: %ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, old_cnt + 1, reason, file, line); +} + +static void pi_unref_dbg(grpc_exec_ctx *exec_ctx, polling_island *pi, + const char *reason, const char *file, int line) { + long old_cnt = gpr_atm_acq_load(&pi->ref_count); + pi_unref(exec_ctx, pi); + gpr_log(GPR_DEBUG, "Unref pi: %p, old:%ld -> new:%ld (%s) - (%s, %d)", + (void *)pi, old_cnt, (old_cnt - 1), reason, file, line); +} + +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue, + const char *file, int line, + const char *reason) { + if (workqueue != NULL) { + pi_add_ref_dbg((polling_island *)workqueue, reason, file, line); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue, + const char *file, int line, const char *reason) { + if (workqueue != NULL) { + pi_unref_dbg(exec_ctx, (polling_island *)workqueue, reason, file, line); + } +} +#else +static grpc_workqueue *workqueue_ref(grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_add_ref((polling_island *)workqueue); + } + return workqueue; +} + +static void workqueue_unref(grpc_exec_ctx *exec_ctx, + grpc_workqueue *workqueue) { + if (workqueue != NULL) { + pi_unref(exec_ctx, (polling_island *)workqueue); + } +} +#endif + +static void pi_add_ref(polling_island *pi) { + gpr_atm_no_barrier_fetch_add(&pi->ref_count, 1); +} + +static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { + /* If ref count went to zero, delete the polling island. + Note that this deletion not be done under a lock. Once the ref count goes + to zero, we are guaranteed that no one else holds a reference to the + polling island (and that there is no racing pi_add_ref() call either). + + Also, if we are deleting the polling island and the merged_to field is + non-empty, we should remove a ref to the merged_to polling island + */ + if (1 == gpr_atm_full_fetch_add(&pi->ref_count, -1)) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + polling_island_delete(exec_ctx, pi); + if (next != NULL) { + PI_UNREF(exec_ctx, next, "pi_delete"); /* Recursive call */ + } + } +} + +static void worker_node_init(worker_node *node) { + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void push_back_worker_node(worker_node *head, worker_node *node) { + node->next = head; + node->prev = head->prev; + head->prev->next = node; + head->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static void remove_worker_node(worker_node *node) { + node->next->prev = node->prev; + node->prev->next = node->next; + /* If node's next and prev point to itself, the node is considered detached + * from the list*/ + node->next = node->prev = node; +} + +/* Not thread safe. Do under a list-level lock */ +static worker_node *pop_front_worker_node(worker_node *head) { + worker_node *node = head->next; + if (node != head) { + remove_worker_node(node); + } else { + node = NULL; + } + + return node; +} + +/* Returns true if the node's next and prev are pointing to itself (which + indicates that the node is not in the list */ +static bool is_worker_node_detached(worker_node *node) { + return (node->next == node->prev && node->next == node); +} + +/* The caller is expected to hold pi->mu lock before calling this function + */ +static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, + size_t fd_count, bool add_fd_refs, + grpc_error **error) { + int err; + size_t i; + struct epoll_event ev; + char *err_msg; + const char *err_desc = "polling_island_add_fds"; + +#ifdef GRPC_TSAN + /* See the definition of g_epoll_sync for more context */ + gpr_atm_rel_store(&g_epoll_sync, (gpr_atm)0); +#endif /* defined(GRPC_TSAN) */ + + for (i = 0; i < fd_count; i++) { + ev.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); + ev.data.ptr = fds[i]; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, fds[i]->fd, &ev); + + if (err < 0) { + if (errno != EEXIST) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) add fd: %d failed with error: %d (%s)", + pi->epoll_fd, fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + continue; + } + + if (pi->fd_cnt == pi->fd_capacity) { + pi->fd_capacity = GPR_MAX(pi->fd_capacity + 8, pi->fd_cnt * 3 / 2); + pi->fds = gpr_realloc(pi->fds, sizeof(grpc_fd *) * pi->fd_capacity); + } + + pi->fds[pi->fd_cnt++] = fds[i]; + if (add_fd_refs) { + GRPC_FD_REF(fds[i], "polling_island"); + } + } +} + +/* The caller is expected to hold pi->mu before calling this */ +static void polling_island_add_wakeup_fd_locked(polling_island *pi, + grpc_wakeup_fd *wakeup_fd, + grpc_error **error) { + struct epoll_event ev; + int err; + char *err_msg; + const char *err_desc = "polling_island_add_wakeup_fd"; + + ev.events = (uint32_t)(EPOLLIN | EPOLLET); + ev.data.ptr = wakeup_fd; + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_ADD, + GRPC_WAKEUP_FD_GET_READ_FD(wakeup_fd), &ev); + if (err < 0 && errno != EEXIST) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) add wakeup fd: %d failed with " + "error: %d (%s)", + pi->epoll_fd, GRPC_WAKEUP_FD_GET_READ_FD(&global_wakeup_fd), + errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_all_fds_locked(polling_island *pi, + bool remove_fd_refs, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fds"; + + for (i = 0; i < pi->fd_cnt; i++) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, pi->fds[i]->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf(&err_msg, + "epoll_ctl (epoll_fd: %d) delete fds[%zu]: %d failed with " + "error: %d (%s)", + pi->epoll_fd, i, pi->fds[i]->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + + if (remove_fd_refs) { + GRPC_FD_UNREF(pi->fds[i], "polling_island"); + } + } + + pi->fd_cnt = 0; +} + +/* The caller is expected to hold pi->mu lock before calling this function */ +static void polling_island_remove_fd_locked(polling_island *pi, grpc_fd *fd, + bool is_fd_closed, + grpc_error **error) { + int err; + size_t i; + char *err_msg; + const char *err_desc = "polling_island_remove_fd"; + + /* If fd is already closed, then it would have been automatically been removed + from the epoll set */ + if (!is_fd_closed) { + err = epoll_ctl(pi->epoll_fd, EPOLL_CTL_DEL, fd->fd, NULL); + if (err < 0 && errno != ENOENT) { + gpr_asprintf( + &err_msg, + "epoll_ctl (epoll_fd: %d) del fd: %d failed with error: %d (%s)", + pi->epoll_fd, fd->fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + gpr_free(err_msg); + } + } + + for (i = 0; i < pi->fd_cnt; i++) { + if (pi->fds[i] == fd) { + pi->fds[i] = pi->fds[--pi->fd_cnt]; + GRPC_FD_UNREF(fd, "polling_island"); + break; + } + } +} + +/* Might return NULL in case of an error */ +static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, + grpc_fd *initial_fd, + grpc_error **error) { + polling_island *pi = NULL; + const char *err_desc = "polling_island_create"; + + *error = GRPC_ERROR_NONE; + + pi = gpr_malloc(sizeof(*pi)); + pi->workqueue_scheduler.vtable = &workqueue_scheduler_vtable; + gpr_mu_init(&pi->mu); + pi->fd_cnt = 0; + pi->fd_capacity = 0; + pi->fds = NULL; + pi->epoll_fd = -1; + + gpr_mu_init(&pi->workqueue_read_mu); + gpr_mpscq_init(&pi->workqueue_items); + gpr_atm_rel_store(&pi->workqueue_item_count, 0); + + gpr_atm_rel_store(&pi->ref_count, 0); + gpr_atm_rel_store(&pi->poller_count, 0); + gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); + + gpr_mu_init(&pi->worker_list_mu); + worker_node_init(&pi->worker_list_head); + + if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), + err_desc)) { + goto done; + } + + pi->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + + if (pi->epoll_fd < 0) { + append_error(error, GRPC_OS_ERROR(errno, "epoll_create1"), err_desc); + goto done; + } + + polling_island_add_wakeup_fd_locked(pi, &global_wakeup_fd, error); + polling_island_add_wakeup_fd_locked(pi, &pi->workqueue_wakeup_fd, error); + + if (initial_fd != NULL) { + polling_island_add_fds_locked(pi, &initial_fd, 1, true, error); + } + +done: + if (*error != GRPC_ERROR_NONE) { + polling_island_delete(exec_ctx, pi); + pi = NULL; + } + return pi; +} + +static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { + GPR_ASSERT(pi->fd_cnt == 0); + + if (pi->epoll_fd >= 0) { + close(pi->epoll_fd); + } + GPR_ASSERT(gpr_atm_no_barrier_load(&pi->workqueue_item_count) == 0); + gpr_mu_destroy(&pi->workqueue_read_mu); + gpr_mpscq_destroy(&pi->workqueue_items); + gpr_mu_destroy(&pi->mu); + grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); + gpr_mu_destroy(&pi->worker_list_mu); + GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); + + gpr_free(pi->fds); + gpr_free(pi); +} + +/* Attempts to gets the last polling island in the linked list (liked by the + * 'merged_to' field). Since this does not lock the polling island, there are no + * guarantees that the island returned is the last island */ +static polling_island *polling_island_maybe_get_latest(polling_island *pi) { + polling_island *next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + while (next != NULL) { + pi = next; + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + } + + return pi; +} + +/* Gets the lock on the *latest* polling island i.e the last polling island in + the linked list (linked by the 'merged_to' field). Call gpr_mu_unlock on the + returned polling island's mu. + Usage: To lock/unlock polling island "pi", do the following: + polling_island *pi_latest = polling_island_lock(pi); + ... + ... critical section .. + ... + gpr_mu_unlock(&pi_latest->mu); // NOTE: use pi_latest->mu. NOT pi->mu */ +static polling_island *polling_island_lock(polling_island *pi) { + polling_island *next = NULL; + + while (true) { + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* Looks like 'pi' is the last node in the linked list but unless we check + this by holding the pi->mu lock, we cannot be sure (i.e without the + pi->mu lock, we don't prevent island merges). + To be absolutely sure, check once more by holding the pi->mu lock */ + gpr_mu_lock(&pi->mu); + next = (polling_island *)gpr_atm_acq_load(&pi->merged_to); + if (next == NULL) { + /* pi is infact the last node and we have the pi->mu lock. we're done */ + break; + } + + /* pi->merged_to is not NULL i.e pi isn't the last node anymore. pi->mu + * isn't the lock we are interested in. Continue traversing the list */ + gpr_mu_unlock(&pi->mu); + } + + pi = next; + } + + return pi; +} + +/* Gets the lock on the *latest* polling islands in the linked lists pointed by + *p and *q (and also updates *p and *q to point to the latest polling islands) + + This function is needed because calling the following block of code to obtain + locks on polling islands (*p and *q) is prone to deadlocks. + { + polling_island_lock(*p, true); + polling_island_lock(*q, true); + } + + Usage/example: + polling_island *p1; + polling_island *p2; + .. + polling_island_lock_pair(&p1, &p2); + .. + .. Critical section with both p1 and p2 locked + .. + // Release locks: Always call polling_island_unlock_pair() to release locks + polling_island_unlock_pair(p1, p2); +*/ +static void polling_island_lock_pair(polling_island **p, polling_island **q) { + polling_island *pi_1 = *p; + polling_island *pi_2 = *q; + polling_island *next_1 = NULL; + polling_island *next_2 = NULL; + + /* The algorithm is simple: + - Go to the last polling islands in the linked lists *pi_1 and *pi_2 (and + keep updating pi_1 and pi_2) + - Then obtain locks on the islands by following a lock order rule of + locking polling_island with lower address first + Special case: Before obtaining the locks, check if pi_1 and pi_2 are + pointing to the same island. If that is the case, we can just call + polling_island_lock() + - After obtaining both the locks, double check that the polling islands + are still the last polling islands in their respective linked lists + (this is because there might have been polling island merges before + we got the lock) + - If the polling islands are the last islands, we are done. If not, + release the locks and continue the process from the first step */ + while (true) { + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + while (next_1 != NULL) { + pi_1 = next_1; + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + } + + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + while (next_2 != NULL) { + pi_2 = next_2; + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + } + + if (pi_1 == pi_2) { + pi_1 = pi_2 = polling_island_lock(pi_1); + break; + } + + if (pi_1 < pi_2) { + gpr_mu_lock(&pi_1->mu); + gpr_mu_lock(&pi_2->mu); + } else { + gpr_mu_lock(&pi_2->mu); + gpr_mu_lock(&pi_1->mu); + } + + next_1 = (polling_island *)gpr_atm_acq_load(&pi_1->merged_to); + next_2 = (polling_island *)gpr_atm_acq_load(&pi_2->merged_to); + if (next_1 == NULL && next_2 == NULL) { + break; + } + + gpr_mu_unlock(&pi_1->mu); + gpr_mu_unlock(&pi_2->mu); + } + + *p = pi_1; + *q = pi_2; +} + +static void polling_island_unlock_pair(polling_island *p, polling_island *q) { + if (p == q) { + gpr_mu_unlock(&p->mu); + } else { + gpr_mu_unlock(&p->mu); + gpr_mu_unlock(&q->mu); + } +} + +static void workqueue_maybe_wakeup(polling_island *pi) { + /* If this thread is the current poller, then it may be that it's about to + decrement the current poller count, so we need to look past this thread */ + bool is_current_poller = (g_current_thread_polling_island == pi); + gpr_atm min_current_pollers_for_wakeup = is_current_poller ? 1 : 0; + gpr_atm current_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + /* Only issue a wakeup if it's likely that some poller could come in and take + it right now. Note that since we do an anticipatory mpscq_pop every poll + loop, it's ok if we miss the wakeup here, as we'll get the work item when + the next poller enters anyway. */ + if (current_pollers > min_current_pollers_for_wakeup) { + GRPC_LOG_IF_ERROR("workqueue_wakeup_fd", + grpc_wakeup_fd_wakeup(&pi->workqueue_wakeup_fd)); + } +} + +static void workqueue_move_items_to_parent(polling_island *q) { + polling_island *p = (polling_island *)gpr_atm_no_barrier_load(&q->merged_to); + if (p == NULL) { + return; + } + gpr_mu_lock(&q->workqueue_read_mu); + int num_added = 0; + while (gpr_atm_no_barrier_load(&q->workqueue_item_count) > 0) { + gpr_mpscq_node *n = gpr_mpscq_pop(&q->workqueue_items); + if (n != NULL) { + gpr_atm_no_barrier_fetch_add(&q->workqueue_item_count, -1); + gpr_atm_no_barrier_fetch_add(&p->workqueue_item_count, 1); + gpr_mpscq_push(&p->workqueue_items, n); + num_added++; + } + } + gpr_mu_unlock(&q->workqueue_read_mu); + if (num_added > 0) { + workqueue_maybe_wakeup(p); + } + workqueue_move_items_to_parent(p); +} + +static polling_island *polling_island_merge(polling_island *p, + polling_island *q, + grpc_error **error) { + /* Get locks on both the polling islands */ + polling_island_lock_pair(&p, &q); + + if (p != q) { + /* Make sure that p points to the polling island with fewer fds than q */ + if (p->fd_cnt > q->fd_cnt) { + GPR_SWAP(polling_island *, p, q); + } + + /* Merge p with q i.e move all the fds from p (The one with fewer fds) to q + Note that the refcounts on the fds being moved will not change here. + This is why the last param in the following two functions is 'false') */ + polling_island_add_fds_locked(q, p->fds, p->fd_cnt, false, error); + polling_island_remove_all_fds_locked(p, false, error); + + /* Wakeup all the pollers (if any) on p so that they pickup this change */ + polling_island_add_wakeup_fd_locked(p, &polling_island_wakeup_fd, error); + + /* Add the 'merged_to' link from p --> q */ + gpr_atm_rel_store(&p->merged_to, (gpr_atm)q); + PI_ADD_REF(q, "pi_merge"); /* To account for the new incoming ref from p */ + + workqueue_move_items_to_parent(p); + } + /* else if p == q, nothing needs to be done */ + + polling_island_unlock_pair(p, q); + + /* Return the merged polling island (Note that no merge would have happened + if p == q which is ok) */ + return q; +} + +static void workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure, + grpc_error *error) { + GPR_TIMER_BEGIN("workqueue.enqueue", 0); + grpc_workqueue *workqueue = (grpc_workqueue *)closure->scheduler; + /* take a ref to the workqueue: otherwise it can happen that whatever events + * this kicks off ends up destroying the workqueue before this function + * completes */ + GRPC_WORKQUEUE_REF(workqueue, "enqueue"); + polling_island *pi = (polling_island *)workqueue; + gpr_atm last = gpr_atm_no_barrier_fetch_add(&pi->workqueue_item_count, 1); + closure->error_data.error = error; + gpr_mpscq_push(&pi->workqueue_items, &closure->next_data.atm_next); + if (last == 0) { + workqueue_maybe_wakeup(pi); + } + workqueue_move_items_to_parent(pi); + GRPC_WORKQUEUE_UNREF(exec_ctx, workqueue, "enqueue"); + GPR_TIMER_END("workqueue.enqueue", 0); +} + +static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) { + polling_island *pi = (polling_island *)workqueue; + return workqueue == NULL ? grpc_schedule_on_exec_ctx + : &pi->workqueue_scheduler; +} + +static grpc_error *polling_island_global_init() { + grpc_error *error = GRPC_ERROR_NONE; + + error = grpc_wakeup_fd_init(&polling_island_wakeup_fd); + if (error == GRPC_ERROR_NONE) { + error = grpc_wakeup_fd_wakeup(&polling_island_wakeup_fd); + } + + return error; +} + +static void polling_island_global_shutdown() { + grpc_wakeup_fd_destroy(&polling_island_wakeup_fd); +} + +/******************************************************************************* + * Fd Definitions + */ + +/* We need to keep a freelist not because of any concerns of malloc performance + * but instead so that implementations with multiple threads in (for example) + * epoll_wait deal with the race between pollset removal and incoming poll + * notifications. + * + * The problem is that the poller ultimately holds a reference to this + * object, so it is very difficult to know when is safe to free it, at least + * without some expensive synchronization. + * + * If we keep the object freelisted, in the worst case losing this race just + * becomes a spurious read notification on a reused fd. + */ + +/* The alarm system needs to be able to wakeup 'some poller' sometimes + * (specifically when a new alarm needs to be triggered earlier than the next + * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a + * case occurs. */ + +static grpc_fd *fd_freelist = NULL; +static gpr_mu fd_freelist_mu; + +#ifdef GRPC_FD_REF_COUNT_DEBUG +#define REF_BY(fd, n, reason) ref_by(fd, n, reason, __FILE__, __LINE__) +#define UNREF_BY(fd, n, reason) unref_by(fd, n, reason, __FILE__, __LINE__) +static void ref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_log(GPR_DEBUG, "FD %d %p ref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) + n, reason, file, line); +#else +#define REF_BY(fd, n, reason) ref_by(fd, n) +#define UNREF_BY(fd, n, reason) unref_by(fd, n) +static void ref_by(grpc_fd *fd, int n) { +#endif + GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0); +} + +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void unref_by(grpc_fd *fd, int n, const char *reason, const char *file, + int line) { + gpr_atm old; + gpr_log(GPR_DEBUG, "FD %d %p unref %d %ld -> %ld [%s; %s:%d]", fd->fd, + (void *)fd, n, gpr_atm_no_barrier_load(&fd->refst), + gpr_atm_no_barrier_load(&fd->refst) - n, reason, file, line); +#else +static void unref_by(grpc_fd *fd, int n) { + gpr_atm old; +#endif + old = gpr_atm_full_fetch_add(&fd->refst, -n); + if (old == n) { + /* Add the fd to the freelist */ + gpr_mu_lock(&fd_freelist_mu); + fd->freelist_next = fd_freelist; + fd_freelist = fd; + grpc_iomgr_unregister_object(&fd->iomgr_object); + + grpc_lfev_destroy(&fd->read_closure); + grpc_lfev_destroy(&fd->write_closure); + + gpr_mu_unlock(&fd_freelist_mu); + } else { + GPR_ASSERT(old > n); + } +} + +/* Increment refcount by two to avoid changing the orphan bit */ +#ifdef GRPC_FD_REF_COUNT_DEBUG +static void fd_ref(grpc_fd *fd, const char *reason, const char *file, + int line) { + ref_by(fd, 2, reason, file, line); +} + +static void fd_unref(grpc_fd *fd, const char *reason, const char *file, + int line) { + unref_by(fd, 2, reason, file, line); +} +#else +static void fd_ref(grpc_fd *fd) { ref_by(fd, 2); } +static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); } +#endif + +static void fd_global_init(void) { gpr_mu_init(&fd_freelist_mu); } + +static void fd_global_shutdown(void) { + gpr_mu_lock(&fd_freelist_mu); + gpr_mu_unlock(&fd_freelist_mu); + while (fd_freelist != NULL) { + grpc_fd *fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + gpr_mu_destroy(&fd->po.mu); + gpr_free(fd); + } + gpr_mu_destroy(&fd_freelist_mu); +} + +static grpc_fd *fd_create(int fd, const char *name) { + grpc_fd *new_fd = NULL; + + gpr_mu_lock(&fd_freelist_mu); + if (fd_freelist != NULL) { + new_fd = fd_freelist; + fd_freelist = fd_freelist->freelist_next; + } + gpr_mu_unlock(&fd_freelist_mu); + + if (new_fd == NULL) { + new_fd = gpr_malloc(sizeof(grpc_fd)); + gpr_mu_init(&new_fd->po.mu); + } + + /* Note: It is not really needed to get the new_fd->po.mu lock here. If this + * is a newly created fd (or an fd we got from the freelist), no one else + * would be holding a lock to it anyway. */ + gpr_mu_lock(&new_fd->po.mu); + new_fd->po.pi = NULL; +#ifdef PO_DEBUG + new_fd->po.obj_type = POLL_OBJ_FD; +#endif + + gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1); + new_fd->fd = fd; + new_fd->orphaned = false; + grpc_lfev_init(&new_fd->read_closure); + grpc_lfev_init(&new_fd->write_closure); + gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); + + new_fd->freelist_next = NULL; + new_fd->on_done_closure = NULL; + + gpr_mu_unlock(&new_fd->po.mu); + + char *fd_name; + gpr_asprintf(&fd_name, "%s fd=%d", name, fd); + grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name); +#ifdef GRPC_FD_REF_COUNT_DEBUG + gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, (void *)new_fd, fd_name); +#endif + gpr_free(fd_name); + return new_fd; +} + +static int fd_wrapped_fd(grpc_fd *fd) { + int ret_fd = -1; + gpr_mu_lock(&fd->po.mu); + if (!fd->orphaned) { + ret_fd = fd->fd; + } + gpr_mu_unlock(&fd->po.mu); + + return ret_fd; +} + +static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *on_done, int *release_fd, + const char *reason) { + bool is_fd_closed = false; + grpc_error *error = GRPC_ERROR_NONE; + polling_island *unref_pi = NULL; + + gpr_mu_lock(&fd->po.mu); + fd->on_done_closure = on_done; + + /* If release_fd is not NULL, we should be relinquishing control of the file + descriptor fd->fd (but we still own the grpc_fd structure). */ + if (release_fd != NULL) { + *release_fd = fd->fd; + } else { + close(fd->fd); + is_fd_closed = true; + } + + fd->orphaned = true; + + /* Remove the active status but keep referenced. We want this grpc_fd struct + to be alive (and not added to freelist) until the end of this function */ + REF_BY(fd, 1, reason); + + /* Remove the fd from the polling island: + - Get a lock on the latest polling island (i.e the last island in the + linked list pointed by fd->po.pi). This is the island that + would actually contain the fd + - Remove the fd from the latest polling island + - Unlock the latest polling island + - Set fd->po.pi to NULL (but remove the ref on the polling island + before doing this.) */ + if (fd->po.pi != NULL) { + polling_island *pi_latest = polling_island_lock(fd->po.pi); + polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error); + gpr_mu_unlock(&pi_latest->mu); + + unref_pi = fd->po.pi; + fd->po.pi = NULL; + } + + grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error)); + + gpr_mu_unlock(&fd->po.mu); + UNREF_BY(fd, 2, reason); /* Drop the reference */ + if (unref_pi != NULL) { + /* Unref stale polling island here, outside the fd lock above. + The polling island owns a workqueue which owns an fd, and unreffing + inside the lock can cause an eventual lock loop that makes TSAN very + unhappy. */ + PI_UNREF(exec_ctx, unref_pi, "fd_orphan"); + } + GRPC_LOG_IF_ERROR("fd_orphan", GRPC_ERROR_REF(error)); + GRPC_ERROR_UNREF(error); +} + +static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx, + grpc_fd *fd) { + gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); + return (grpc_pollset *)notifier; +} + +static bool fd_is_shutdown(grpc_fd *fd) { + return grpc_lfev_is_shutdown(&fd->read_closure); +} + +/* Might be called multiple times */ +static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_error *why) { + if (grpc_lfev_set_shutdown(exec_ctx, &fd->read_closure, + GRPC_ERROR_REF(why))) { + shutdown(fd->fd, SHUT_RDWR); + grpc_lfev_set_shutdown(exec_ctx, &fd->write_closure, GRPC_ERROR_REF(why)); + } + GRPC_ERROR_UNREF(why); +} + +static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->read_closure, closure); +} + +static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_closure *closure) { + grpc_lfev_notify_on(exec_ctx, &fd->write_closure, closure); +} + +static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) { + gpr_mu_lock(&fd->po.mu); + grpc_workqueue *workqueue = + GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue"); + gpr_mu_unlock(&fd->po.mu); + return workqueue; +} + +/******************************************************************************* + * Pollset Definitions + */ +GPR_TLS_DECL(g_current_thread_pollset); +GPR_TLS_DECL(g_current_thread_worker); +static __thread bool g_initialized_sigmask; +static __thread sigset_t g_orig_sigmask; +static __thread sigset_t g_wakeup_sig_set; + +static void sig_handler(int sig_num) { +#ifdef GRPC_EPOLL_DEBUG + gpr_log(GPR_INFO, "Received signal %d", sig_num); +#endif +} + +static void pollset_worker_init(grpc_pollset_worker *worker) { + worker->pt_id = pthread_self(); + worker->next = worker->prev = NULL; + gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); + gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); + worker_node_init(&worker->pi_list_link); +} + +static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } + +/* Global state management */ +static grpc_error *pollset_global_init(void) { + gpr_tls_init(&g_current_thread_pollset); + gpr_tls_init(&g_current_thread_worker); + poller_kick_init(); + return grpc_wakeup_fd_init(&global_wakeup_fd); +} + +static void pollset_global_shutdown(void) { + grpc_wakeup_fd_destroy(&global_wakeup_fd); + gpr_tls_destroy(&g_current_thread_pollset); + gpr_tls_destroy(&g_current_thread_worker); +} + +static grpc_error *worker_kick(grpc_pollset_worker *worker, + gpr_atm *is_kicked) { + grpc_error *err = GRPC_ERROR_NONE; + + /* Kick the worker only if it was not already kicked */ + if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { + GRPC_POLLING_TRACE( + "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", + (void *)worker, (long int)worker->pt_id); + int err_num = pthread_kill(worker->pt_id, grpc_wakeup_signal); + if (err_num != 0) { + err = GRPC_OS_ERROR(err_num, "pthread_kill"); + } + } + return err; +} + +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_kicked); +} + +static grpc_error *poller_kick(grpc_pollset_worker *worker) { + return worker_kick(worker, &worker->is_polling_turn); +} + +/* Return 1 if the pollset has active threads in pollset_work (pollset must + * be locked) */ +static int pollset_has_workers(grpc_pollset *p) { + return p->root_worker.next != &p->root_worker; +} + +static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev->next = worker->next; + worker->next->prev = worker->prev; +} + +static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { + if (pollset_has_workers(p)) { + grpc_pollset_worker *w = p->root_worker.next; + remove_worker(p, w); + return w; + } else { + return NULL; + } +} + +static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->next = &p->root_worker; + worker->prev = worker->next->prev; + worker->prev->next = worker->next->prev = worker; +} + +static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { + worker->prev = &p->root_worker; + worker->next = worker->prev->next; + worker->prev->next = worker->next->prev = worker; +} + +/* p->mu must be held before calling this function */ +static grpc_error *pollset_kick(grpc_pollset *p, + grpc_pollset_worker *specific_worker) { + GPR_TIMER_BEGIN("pollset_kick", 0); + grpc_error *error = GRPC_ERROR_NONE; + const char *err_desc = "Kick Failure"; + grpc_pollset_worker *worker = specific_worker; + if (worker != NULL) { + if (worker == GRPC_POLLSET_KICK_BROADCAST) { + if (pollset_has_workers(p)) { + GPR_TIMER_BEGIN("pollset_kick.broadcast", 0); + for (worker = p->root_worker.next; worker != &p->root_worker; + worker = worker->next) { + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + GPR_TIMER_END("pollset_kick.broadcast", 0); + } else { + p->kicked_without_pollers = true; + } + } else { + GPR_TIMER_MARK("kicked_specifically", 0); + if (gpr_tls_get(&g_current_thread_worker) != (intptr_t)worker) { + append_error(&error, pollset_worker_kick(worker), err_desc); + } + } + } else if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)p) { + /* Since worker == NULL, it means that we can kick "any" worker on this + pollset 'p'. If 'p' happens to be the same pollset this thread is + currently polling (i.e in pollset_work() function), then there is no need + to kick any other worker since the current thread can just absorb the + kick. This is the reason why we enter this case only when + g_current_thread_pollset is != p */ + + GPR_TIMER_MARK("kick_anonymous", 0); + worker = pop_front_worker(p); + if (worker != NULL) { + GPR_TIMER_MARK("finally_kick", 0); + push_back_worker(p, worker); + append_error(&error, pollset_worker_kick(worker), err_desc); + } else { + GPR_TIMER_MARK("kicked_no_pollers", 0); + p->kicked_without_pollers = true; + } + } + + GPR_TIMER_END("pollset_kick", 0); + GRPC_LOG_IF_ERROR("pollset_kick", GRPC_ERROR_REF(error)); + return error; +} + +static grpc_error *kick_poller(void) { + return grpc_wakeup_fd_wakeup(&global_wakeup_fd); +} + +static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { + gpr_mu_init(&pollset->po.mu); + *mu = &pollset->po.mu; + pollset->po.pi = NULL; +#ifdef PO_DEBUG + pollset->po.obj_type = POLL_OBJ_POLLSET; +#endif + + pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; + pollset->kicked_without_pollers = false; + + pollset->shutting_down = false; + pollset->finish_shutdown_called = false; + pollset->shutdown_done = NULL; +} + +/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ +static struct timespec millis_to_timespec(int millis) { + struct timespec linux_ts; + gpr_timespec gpr_ts; + + if (millis == -1) { + gpr_ts = gpr_inf_future(GPR_TIMESPAN); + } else { + gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); + } + + linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; + linux_ts.tv_nsec = gpr_ts.tv_nsec; + return linux_ts; +} + +/* Convert a timespec to milliseconds: + - Very small or negative poll times are clamped to zero to do a non-blocking + poll (which becomes spin polling) + - Other small values are rounded up to one millisecond + - Longer than a millisecond polls are rounded up to the next nearest + millisecond to avoid spinning + - Infinite timeouts are converted to -1 */ +static int poll_deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int64_t max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return -1; + } + + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + int millis = gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); + return millis >= 1 ? millis : 1; +} + +static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + grpc_pollset *notifier) { + grpc_lfev_set_ready(exec_ctx, &fd->read_closure); + + /* Note, it is possible that fd_become_readable might be called twice with + different 'notifier's when an fd becomes readable and it is in two epoll + sets (This can happen briefly during polling island merges). In such cases + it does not really matter which notifer is set as the read_notifier_pollset + (They would both point to the same polling island anyway) */ + /* Use release store to match with acquire load in fd_get_read_notifier */ + gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); +} + +static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) { + grpc_lfev_set_ready(exec_ctx, &fd->write_closure); +} + +static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx, + grpc_pollset *ps, char *reason) { + if (ps->po.pi != NULL) { + PI_UNREF(exec_ctx, ps->po.pi, reason); + } + ps->po.pi = NULL; +} + +static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset) { + /* The pollset cannot have any workers if we are at this stage */ + GPR_ASSERT(!pollset_has_workers(pollset)); + + pollset->finish_shutdown_called = true; + + /* Release the ref and set pollset->po.pi to NULL */ + pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown"); + grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE); +} + +/* pollset->po.mu lock must be held by the caller before calling this */ +static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure) { + GPR_TIMER_BEGIN("pollset_shutdown", 0); + GPR_ASSERT(!pollset->shutting_down); + pollset->shutting_down = true; + pollset->shutdown_done = closure; + pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + + /* If the pollset has any workers, we cannot call finish_shutdown_locked() + because it would release the underlying polling island. In such a case, we + let the last worker call finish_shutdown_locked() from pollset_work() */ + if (!pollset_has_workers(pollset)) { + GPR_ASSERT(!pollset->finish_shutdown_called); + GPR_TIMER_MARK("pollset_shutdown.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + } + GPR_TIMER_END("pollset_shutdown", 0); +} + +/* pollset_shutdown is guaranteed to be called before pollset_destroy. So other + * than destroying the mutexes, there is nothing special that needs to be done + * here */ +static void pollset_destroy(grpc_pollset *pollset) { + GPR_ASSERT(!pollset_has_workers(pollset)); + gpr_mu_destroy(&pollset->po.mu); +} + +static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, + polling_island *pi) { + if (gpr_mu_trylock(&pi->workqueue_read_mu)) { + gpr_mpscq_node *n = gpr_mpscq_pop(&pi->workqueue_items); + gpr_mu_unlock(&pi->workqueue_read_mu); + if (n != NULL) { + if (gpr_atm_full_fetch_add(&pi->workqueue_item_count, -1) > 1) { + workqueue_maybe_wakeup(pi); + } + grpc_closure *c = (grpc_closure *)n; + grpc_error *error = c->error_data.error; +#ifndef NDEBUG + c->scheduled = false; +#endif + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + return true; + } else if (gpr_atm_no_barrier_load(&pi->workqueue_item_count) > 0) { + /* n == NULL might mean there's work but it's not available to be popped + * yet - try to ensure another workqueue wakes up to check shortly if so + */ + workqueue_maybe_wakeup(pi); + } + } + return false; +} + +/* NOTE: This function may modify 'now' */ +static bool acquire_polling_lease(grpc_pollset_worker *worker, + polling_island *pi, gpr_timespec deadline, + gpr_timespec *now) { + bool is_lease_acquired = false; + + gpr_mu_lock(&pi->worker_list_mu); // LOCK + long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + + if (num_pollers >= g_max_pollers_per_pi) { + push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + + bool is_timeout = false; + int ret; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); + if (timeout_ms == -1) { + ret = sigwaitinfo(&g_wakeup_sig_set, NULL); + } else { + struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); + GRPC_SCHEDULING_START_BLOCKING_REGION; + ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); + GRPC_SCHEDULING_END_BLOCKING_REGION; + } + + if (ret == -1) { + if (errno == EAGAIN) { + is_timeout = true; + } else { + /* NOTE: This should not happen. If we see these log messages, it means + we are most likely doing something incorrect in the setup * needed + for sigwaitinfo/sigtimedwait */ + gpr_log(GPR_ERROR, + "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, + timeout_ms); + } + } + + /* Did the worker come out of sigtimedwait due to a thread that just + exited epoll and kicking it (in release_polling_lease function). */ + bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); + + /* Did the worker come out of sigtimedwait due to a thread alerting it that + some completion event was (likely) available in the completion queue */ + bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); + + if (is_kicked || is_timeout) { + *now = deadline; /* Essentially make the epoll timeout = 0 */ + } else if (is_polling_turn) { + *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ + } + + gpr_mu_lock(&pi->worker_list_mu); // LOCK + /* The node might have already been removed from the list by the poller + that kicked this. However it is safe to call 'remove_worker_node' on + an already detached node */ + remove_worker_node(&worker->pi_list_link); + /* It is important to read the num_pollers again under the lock so that we + * have the latest num_pollers value that doesn't change while we are doing + * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ + num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); + } + + if (num_pollers < g_max_pollers_per_pi) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); + is_lease_acquired = true; + } + + gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK + return is_lease_acquired; +} + +static void release_polling_lease(polling_island *pi, grpc_error **error) { + gpr_mu_lock(&pi->worker_list_mu); + + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); + worker_node *node = pop_front_worker_node(&pi->worker_list_head); + if (node != NULL) { + grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); + append_error(error, poller_kick(next_worker), "poller kick error"); + } + + gpr_mu_unlock(&pi->worker_list_mu); +} + +#define GRPC_EPOLL_MAX_EVENTS 100 +static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, + grpc_pollset *pollset, polling_island *pi, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + /* Only g_max_pollers_per_pi threads can be doing polling in parallel. + If we cannot get a lease, we cannot continue to do epoll_pwait() */ + if (!acquire_polling_lease(worker, pi, deadline, &now)) { + return; + } + + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; + int ep_rv; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; + + /* timeout_ms is the time between 'now' and 'deadline' */ + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = + epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + + /* Give back the lease right away so that some other thread can enter */ + release_polling_lease(pi, error); + + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } +} + +/* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ +static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker *worker, + gpr_timespec now, gpr_timespec deadline, + sigset_t *sig_mask, grpc_error **error) { + int epoll_fd = -1; + polling_island *pi = NULL; + GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); + + /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the + latest polling island pointed by pollset->po.pi + + Since epoll_fd is immutable, it is safe to read it without a lock on the + polling island. There is however a possibility that the polling island from + which we got the epoll_fd, got merged with another island in the meantime. + This is okay because in such a case, we will wakeup right-away from + epoll_pwait() (because any merge will poison the old polling island's epoll + set 'polling_island_wakeup_fd') and then pick up the latest polling_island + the next time this function - pollset_work_and_unlock()) is called */ + + if (pollset->po.pi == NULL) { + pollset->po.pi = polling_island_create(exec_ctx, NULL, error); + if (pollset->po.pi == NULL) { + GPR_TIMER_END("pollset_work_and_unlock", 0); + return; /* Fatal error. Cannot continue */ + } + + PI_ADD_REF(pollset->po.pi, "ps"); + GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p", + (void *)pollset, (void *)pollset->po.pi); + } + + pi = polling_island_maybe_get_latest(pollset->po.pi); + epoll_fd = pi->epoll_fd; + + /* Update the pollset->po.pi since the island being pointed by + pollset->po.pi maybe older than the one pointed by pi) */ + if (pollset->po.pi != pi) { + /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the + polling island to be deleted */ + PI_ADD_REF(pi, "ps"); + PI_UNREF(exec_ctx, pollset->po.pi, "ps"); + pollset->po.pi = pi; + } + + /* Add an extra ref so that the island does not get destroyed (which means + the epoll_fd won't be closed) while we are are doing an epoll_wait() on the + epoll_fd */ + PI_ADD_REF(pi, "ps_work"); + gpr_mu_unlock(&pollset->po.mu); + + /* If we get some workqueue work to do, it might end up completing an item on + the completion queue, so there's no need to poll... so we skip that and + redo the complete loop to verify */ + if (!maybe_do_workqueue_work(exec_ctx, pi)) { + g_current_thread_polling_island = pi; + pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, + deadline, sig_mask, error); + g_current_thread_polling_island = NULL; + } + + GPR_ASSERT(pi != NULL); + + /* Before leaving, release the extra ref we added to the polling island. It + is important to use "pi" here (i.e our old copy of pollset->po.pi + that we got before releasing the polling island lock). This is because + pollset->po.pi pointer might get udpated in other parts of the + code when there is an island merge while we are doing epoll_wait() above */ + PI_UNREF(exec_ctx, pi, "ps_work"); + + GPR_TIMER_END("pollset_work_and_unlock", 0); +} + +/* pollset->po.mu lock must be held by the caller before calling this. + The function pollset_work() may temporarily release the lock (pollset->po.mu) + during the course of its execution but it will always re-acquire the lock and + ensure that it is held by the time the function returns */ +static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker_hdl, + gpr_timespec now, gpr_timespec deadline) { + GPR_TIMER_BEGIN("pollset_work", 0); + grpc_error *error = GRPC_ERROR_NONE; + + grpc_pollset_worker worker; + pollset_worker_init(&worker); + + if (worker_hdl) *worker_hdl = &worker; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); + gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); + + if (pollset->kicked_without_pollers) { + /* If the pollset was kicked without pollers, pretend that the current + worker got the kick and skip polling. A kick indicates that there is some + work that needs attention like an event on the completion queue or an + alarm */ + GPR_TIMER_MARK("pollset_work.kicked_without_pollers", 0); + pollset->kicked_without_pollers = 0; + } else if (!pollset->shutting_down) { + /* We use the posix-signal with number 'grpc_wakeup_signal' for waking up + (i.e 'kicking') a worker in the pollset. A 'kick' is a way to inform the + worker that there is some pending work that needs immediate attention + (like an event on the completion queue, or a polling island merge that + results in a new epoll-fd to wait on) and that the worker should not + spend time waiting in epoll_pwait(). + + A worker can be kicked anytime from the point it is added to the pollset + via push_front_worker() (or push_back_worker()) to the point it is + removed via remove_worker(). + If the worker is kicked before/during it calls epoll_pwait(), it should + immediately exit from epoll_wait(). If the worker is kicked after it + returns from epoll_wait(), then nothing really needs to be done. + + To accomplish this, we mask 'grpc_wakeup_signal' on this thread at all + times *except* when it is in epoll_pwait(). This way, the worker never + misses acting on a kick */ + + if (!g_initialized_sigmask) { + sigemptyset(&g_wakeup_sig_set); + sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); + sigdelset(&g_orig_sigmask, grpc_wakeup_signal); + g_initialized_sigmask = true; + /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. + This is the mask used at all times *except during + epoll_wait()*" + g_orig_sigmask: The thread mask which allows 'grpc_wakeup_signal' and + this is the mask to use *during epoll_wait()* + + The new_mask is set on the worker before it is added to the pollset + (i.e before it can be kicked) */ + } + + push_front_worker(pollset, &worker); /* Add worker to pollset */ + + pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, + &g_orig_sigmask, &error); + grpc_exec_ctx_flush(exec_ctx); + + gpr_mu_lock(&pollset->po.mu); + + /* Note: There is no need to reset worker.is_kicked to 0 since we are no + longer going to use this worker */ + remove_worker(pollset, &worker); + } + + /* If we are the last worker on the pollset (i.e pollset_has_workers() is + false at this point) and the pollset is shutting down, we may have to + finish the shutdown process by calling finish_shutdown_locked(). + See pollset_shutdown() for more details. + + Note: Continuing to access pollset here is safe; it is the caller's + responsibility to not destroy a pollset when it has outstanding calls to + pollset_work() */ + if (pollset->shutting_down && !pollset_has_workers(pollset) && + !pollset->finish_shutdown_called) { + GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0); + finish_shutdown_locked(exec_ctx, pollset); + + gpr_mu_unlock(&pollset->po.mu); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&pollset->po.mu); + } + + if (worker_hdl) *worker_hdl = NULL; + + gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); + gpr_tls_set(&g_current_thread_worker, (intptr_t)0); + + GPR_TIMER_END("pollset_work", 0); + + GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); + return error; +} + +static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag, + poll_obj_type bag_type, poll_obj *item, + poll_obj_type item_type) { + GPR_TIMER_BEGIN("add_poll_object", 0); + +#ifdef PO_DEBUG + GPR_ASSERT(item->obj_type == item_type); + GPR_ASSERT(bag->obj_type == bag_type); +#endif + + grpc_error *error = GRPC_ERROR_NONE; + polling_island *pi_new = NULL; + + gpr_mu_lock(&bag->mu); + gpr_mu_lock(&item->mu); + +retry: + /* + * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing + * 2) If item->pi and bag->pi are both NULL, create a new polling island (with + * a refcount of 2) and point item->pi and bag->pi to the new island + * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to + * the other's non-NULL pi + * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the + * polling islands and update item->pi and bag->pi to point to the new + * island + */ + + /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already + * orphaned */ + if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) { + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + return; + } + + if (item->pi == bag->pi) { + pi_new = item->pi; + if (pi_new == NULL) { + /* GPR_ASSERT(item->pi == bag->pi == NULL) */ + + /* If we are adding an fd to a bag (i.e pollset or pollset_set), then + * we need to do some extra work to make TSAN happy */ + if (item_type == POLL_OBJ_FD) { + /* Unlock before creating a new polling island: the polling island will + create a workqueue which creates a file descriptor, and holding an fd + lock here can eventually cause a loop to appear to TSAN (making it + unhappy). We don't think it's a real loop (there's an epoch point + where that loop possibility disappears), but the advantages of + keeping TSAN happy outweigh any performance advantage we might have + by keeping the lock held. */ + gpr_mu_unlock(&item->mu); + pi_new = polling_island_create(exec_ctx, FD_FROM_PO(item), &error); + gpr_mu_lock(&item->mu); + + /* Need to reverify any assumptions made between the initial lock and + getting to this branch: if they've changed, we need to throw away our + work and figure things out again. */ + if (item->pi != NULL) { + GRPC_POLLING_TRACE( + "add_poll_object: Raced creating new polling island. pi_new: %p " + "(fd: %d, %s: %p)", + (void *)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type), + (void *)bag); + /* No need to lock 'pi_new' here since this is a new polling island + and no one has a reference to it yet */ + polling_island_remove_all_fds_locked(pi_new, true, &error); + + /* Ref and unref so that the polling island gets deleted during unref + */ + PI_ADD_REF(pi_new, "dance_of_destruction"); + PI_UNREF(exec_ctx, pi_new, "dance_of_destruction"); + goto retry; + } + } else { + pi_new = polling_island_create(exec_ctx, NULL, &error); + } + + GRPC_POLLING_TRACE( + "add_poll_object: Created new polling island. pi_new: %p (%s: %p, " + "%s: %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + GRPC_POLLING_TRACE( + "add_poll_object: Same polling island. pi: %p (%s, %s)", + (void *)pi_new, poll_obj_string(item_type), + poll_obj_string(bag_type)); + } + } else if (item->pi == NULL) { + /* GPR_ASSERT(bag->pi != NULL) */ + /* Make pi_new point to latest pi*/ + pi_new = polling_island_lock(bag->pi); + + if (item_type == POLL_OBJ_FD) { + grpc_fd *fd = FD_FROM_PO(item); + polling_island_add_fds_locked(pi_new, &fd, 1, true, &error); + } + + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else if (bag->pi == NULL) { + /* GPR_ASSERT(item->pi != NULL) */ + /* Make pi_new to point to latest pi */ + pi_new = polling_island_lock(item->pi); + gpr_mu_unlock(&pi_new->mu); + GRPC_POLLING_TRACE( + "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } else { + pi_new = polling_island_merge(item->pi, bag->pi, &error); + GRPC_POLLING_TRACE( + "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, " + "bag(%s): %p)", + (void *)pi_new, poll_obj_string(item_type), (void *)item, + poll_obj_string(bag_type), (void *)bag); + } + + /* At this point, pi_new is the polling island that both item->pi and bag->pi + MUST be pointing to */ + + if (item->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(item_type)); + if (item->pi != NULL) { + PI_UNREF(exec_ctx, item->pi, poll_obj_string(item_type)); + } + item->pi = pi_new; + } + + if (bag->pi != pi_new) { + PI_ADD_REF(pi_new, poll_obj_string(bag_type)); + if (bag->pi != NULL) { + PI_UNREF(exec_ctx, bag->pi, poll_obj_string(bag_type)); + } + bag->pi = pi_new; + } + + gpr_mu_unlock(&item->mu); + gpr_mu_unlock(&bag->mu); + + GRPC_LOG_IF_ERROR("add_poll_object", error); + GPR_TIMER_END("add_poll_object", 0); +} + +static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pollset->po, POLL_OBJ_POLLSET, &fd->po, + POLL_OBJ_FD); +} + +/******************************************************************************* + * Pollset-set Definitions + */ + +static grpc_pollset_set *pollset_set_create(void) { + grpc_pollset_set *pss = gpr_malloc(sizeof(*pss)); + gpr_mu_init(&pss->po.mu); + pss->po.pi = NULL; +#ifdef PO_DEBUG + pss->po.obj_type = POLL_OBJ_POLLSET_SET; +#endif + return pss; +} + +static void pollset_set_destroy(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss) { + gpr_mu_destroy(&pss->po.mu); + + if (pss->po.pi != NULL) { + PI_UNREF(exec_ctx, pss->po.pi, "pss_destroy"); + } + + gpr_free(pss); +} + +static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &fd->po, + POLL_OBJ_FD); +} + +static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss, + grpc_fd *fd) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &ps->po, + POLL_OBJ_POLLSET); +} + +static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *pss, grpc_pollset *ps) { + /* Nothing to do */ +} + +static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + add_poll_object(exec_ctx, &bag->po, POLL_OBJ_POLLSET_SET, &item->po, + POLL_OBJ_POLLSET_SET); +} + +static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx, + grpc_pollset_set *bag, + grpc_pollset_set *item) { + /* Nothing to do */ +} + +/******************************************************************************* + * Event engine binding + */ + +static void shutdown_engine(void) { + fd_global_shutdown(); + pollset_global_shutdown(); + polling_island_global_shutdown(); +} + +static const grpc_event_engine_vtable vtable = { + .pollset_size = sizeof(grpc_pollset), + + .fd_create = fd_create, + .fd_wrapped_fd = fd_wrapped_fd, + .fd_orphan = fd_orphan, + .fd_shutdown = fd_shutdown, + .fd_is_shutdown = fd_is_shutdown, + .fd_notify_on_read = fd_notify_on_read, + .fd_notify_on_write = fd_notify_on_write, + .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset, + .fd_get_workqueue = fd_get_workqueue, + + .pollset_init = pollset_init, + .pollset_shutdown = pollset_shutdown, + .pollset_destroy = pollset_destroy, + .pollset_work = pollset_work, + .pollset_kick = pollset_kick, + .pollset_add_fd = pollset_add_fd, + + .pollset_set_create = pollset_set_create, + .pollset_set_destroy = pollset_set_destroy, + .pollset_set_add_pollset = pollset_set_add_pollset, + .pollset_set_del_pollset = pollset_set_del_pollset, + .pollset_set_add_pollset_set = pollset_set_add_pollset_set, + .pollset_set_del_pollset_set = pollset_set_del_pollset_set, + .pollset_set_add_fd = pollset_set_add_fd, + .pollset_set_del_fd = pollset_set_del_fd, + + .kick_poller = kick_poller, + + .workqueue_ref = workqueue_ref, + .workqueue_unref = workqueue_unref, + .workqueue_scheduler = workqueue_scheduler, + + .shutdown_engine = shutdown_engine, +}; + +/* It is possible that GLIBC has epoll but the underlying kernel doesn't. + * Create a dummy epoll_fd to make sure epoll support is available */ +static bool is_epoll_available() { + int fd = epoll_create1(EPOLL_CLOEXEC); + if (fd < 0) { + gpr_log( + GPR_ERROR, + "epoll_create1 failed with error: %d. Not using epoll polling engine", + fd); + return false; + } + close(fd); + return true; +} + +/* This is mainly for testing purposes. Checks to see if environment variable + * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to + * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ +static void set_max_pollers_per_island() { + char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); + if (s) { + g_max_pollers_per_pi = (int)strtol(s, NULL, 10); + if (g_max_pollers_per_pi < 0) { + g_max_pollers_per_pi = INT_MAX; + } + } else { + g_max_pollers_per_pi = INT_MAX; + } + + gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", + g_max_pollers_per_pi); +} + +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux(void) { + /* If use of signals is disabled, we cannot use epoll engine*/ + if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { + return NULL; + } + + if (!grpc_has_wakeup_fd()) { + return NULL; + } + + if (!is_epoll_available()) { + return NULL; + } + + if (!is_grpc_wakeup_signal_initialized) { + grpc_use_signal(SIGRTMIN + 6); + } + + set_max_pollers_per_island(); + + fd_global_init(); + + if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { + return NULL; + } + + if (!GRPC_LOG_IF_ERROR("polling_island_global_init", + polling_island_global_init())) { + return NULL; + } + + return &vtable; +} + +#else /* defined(GRPC_LINUX_EPOLL) */ +#if defined(GRPC_POSIX_SOCKET) +#include "src/core/lib/iomgr/ev_posix.h" +/* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return + * NULL */ +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux(void) { + return NULL; +} +#endif /* defined(GRPC_POSIX_SOCKET) */ +#endif /* !defined(GRPC_LINUX_EPOLL) */ diff --git a/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h new file mode 100644 index 00000000000..23ff8570c1b --- /dev/null +++ b/src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H +#define GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H + +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" + +const grpc_event_engine_vtable *grpc_init_epoll_limited_pollers_linux(void); + +#endif /* GRPC_CORE_LIB_IOMGR_EV_EPOLL_LIMITED_POLLERS_LINUX_H */ diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 64ea7b192ac..e603a755937 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -40,7 +40,6 @@ #include #include -#include #include #include #include @@ -63,7 +62,6 @@ #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/block_annotate.h" -#include "src/core/lib/support/env.h" /* TODO: sreek - Move this to init.c and initialize this like other tracers. */ static int grpc_polling_trace = 0; /* Disabled by default */ @@ -75,10 +73,6 @@ static int grpc_polling_trace = 0; /* Disabled by default */ /* Uncomment the following to enable extra checks on poll_object operations */ /* #define PO_DEBUG */ -/* The maximum number of polling threads per polling island. By default no - limit */ -static int g_max_pollers_per_pi = INT_MAX; - static int grpc_wakeup_signal = -1; static bool is_grpc_wakeup_signal_initialized = false; @@ -201,11 +195,6 @@ static void fd_global_shutdown(void); #endif /* !defined(GRPC_PI_REF_COUNT_DEBUG) */ -typedef struct worker_node { - struct worker_node *next; - struct worker_node *prev; -} worker_node; - /* This is also used as grpc_workqueue (by directly casing it) */ typedef struct polling_island { grpc_closure_scheduler workqueue_scheduler; @@ -240,10 +229,6 @@ typedef struct polling_island { /* Wakeup fd used to wake pollers to check the contents of workqueue_items */ grpc_wakeup_fd workqueue_wakeup_fd; - /* The list of workers waiting to do polling on this polling island */ - gpr_mu worker_list_mu; - worker_node worker_list_head; - /* The fd of the underlying epoll set */ int epoll_fd; @@ -256,24 +241,14 @@ typedef struct polling_island { /******************************************************************************* * Pollset Declarations */ -#define WORKER_FROM_WORKER_LIST_NODE(p) \ - (struct grpc_pollset_worker *)(((char *)(p)) - \ - offsetof(grpc_pollset_worker, pi_list_link)) struct grpc_pollset_worker { /* Thread id of this worker */ pthread_t pt_id; /* Used to prevent a worker from getting kicked multiple times */ gpr_atm is_kicked; - struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; - - /* Indicates if it is this worker's turn to do epoll */ - gpr_atm is_polling_turn; - - /* Node in the polling island's worker list. */ - worker_node pi_list_link; }; struct grpc_pollset { @@ -417,47 +392,7 @@ static void pi_unref(grpc_exec_ctx *exec_ctx, polling_island *pi) { } } -static void worker_node_init(worker_node *node) { - node->next = node->prev = node; -} - -/* Not thread safe. Do under a list-level lock */ -static void push_back_worker_node(worker_node *head, worker_node *node) { - node->next = head; - node->prev = head->prev; - head->prev->next = node; - head->prev = node; -} - -/* Not thread safe. Do under a list-level lock */ -static void remove_worker_node(worker_node *node) { - node->next->prev = node->prev; - node->prev->next = node->next; - /* If node's next and prev point to itself, the node is considered detached - * from the list*/ - node->next = node->prev = node; -} - -/* Not thread safe. Do under a list-level lock */ -static worker_node *pop_front_worker_node(worker_node *head) { - worker_node *node = head->next; - if (node != head) { - remove_worker_node(node); - } else { - node = NULL; - } - - return node; -} - -/* Returns true if the node's next and prev are pointing to itself (which - indicates that the node is not in the list */ -static bool is_worker_node_detached(worker_node *node) { - return (node->next == node->prev && node->next == node); -} - -/* The caller is expected to hold pi->mu lock before calling this function - */ +/* The caller is expected to hold pi->mu lock before calling this function */ static void polling_island_add_fds_locked(polling_island *pi, grpc_fd **fds, size_t fd_count, bool add_fd_refs, grpc_error **error) { @@ -611,9 +546,6 @@ static polling_island *polling_island_create(grpc_exec_ctx *exec_ctx, gpr_atm_rel_store(&pi->poller_count, 0); gpr_atm_rel_store(&pi->merged_to, (gpr_atm)NULL); - gpr_mu_init(&pi->worker_list_mu); - worker_node_init(&pi->worker_list_head); - if (!append_error(error, grpc_wakeup_fd_init(&pi->workqueue_wakeup_fd), err_desc)) { goto done; @@ -652,9 +584,6 @@ static void polling_island_delete(grpc_exec_ctx *exec_ctx, polling_island *pi) { gpr_mpscq_destroy(&pi->workqueue_items); gpr_mu_destroy(&pi->mu); grpc_wakeup_fd_destroy(&pi->workqueue_wakeup_fd); - gpr_mu_destroy(&pi->worker_list_mu); - GPR_ASSERT(is_worker_node_detached(&pi->worker_list_head)); - gpr_free(pi->fds); gpr_free(pi); } @@ -1173,7 +1102,6 @@ GPR_TLS_DECL(g_current_thread_pollset); GPR_TLS_DECL(g_current_thread_worker); static __thread bool g_initialized_sigmask; static __thread sigset_t g_orig_sigmask; -static __thread sigset_t g_wakeup_sig_set; static void sig_handler(int sig_num) { #ifdef GRPC_EPOLL_DEBUG @@ -1181,14 +1109,6 @@ static void sig_handler(int sig_num) { #endif } -static void pollset_worker_init(grpc_pollset_worker *worker) { - worker->pt_id = pthread_self(); - worker->next = worker->prev = NULL; - gpr_atm_no_barrier_store(&worker->is_kicked, (gpr_atm)0); - gpr_atm_no_barrier_store(&worker->is_polling_turn, (gpr_atm)0); - worker_node_init(&worker->pi_list_link); -} - static void poller_kick_init() { signal(grpc_wakeup_signal, sig_handler); } /* Global state management */ @@ -1205,12 +1125,11 @@ static void pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_worker); } -static grpc_error *worker_kick(grpc_pollset_worker *worker, - gpr_atm *is_kicked) { +static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { grpc_error *err = GRPC_ERROR_NONE; /* Kick the worker only if it was not already kicked */ - if (gpr_atm_no_barrier_cas(is_kicked, (gpr_atm)0, (gpr_atm)1)) { + if (gpr_atm_no_barrier_cas(&worker->is_kicked, (gpr_atm)0, (gpr_atm)1)) { GRPC_POLLING_TRACE( "pollset_worker_kick: Kicking worker: %p (thread id: %ld)", (void *)worker, (long int)worker->pt_id); @@ -1222,14 +1141,6 @@ static grpc_error *worker_kick(grpc_pollset_worker *worker, return err; } -static grpc_error *pollset_worker_kick(grpc_pollset_worker *worker) { - return worker_kick(worker, &worker->is_kicked); -} - -static grpc_error *poller_kick(grpc_pollset_worker *worker) { - return worker_kick(worker, &worker->is_polling_turn); -} - /* Return 1 if the pollset has active threads in pollset_work (pollset must * be locked) */ static int pollset_has_workers(grpc_pollset *p) { @@ -1335,22 +1246,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) { pollset->shutdown_done = NULL; } -/* Convert millis to timespec (clock-type is assumed to be GPR_TIMESPAN) */ -static struct timespec millis_to_timespec(int millis) { - struct timespec linux_ts; - gpr_timespec gpr_ts; - - if (millis == -1) { - gpr_ts = gpr_inf_future(GPR_TIMESPAN); - } else { - gpr_ts = gpr_time_from_millis(millis, GPR_TIMESPAN); - } - - linux_ts.tv_sec = (time_t)gpr_ts.tv_sec; - linux_ts.tv_nsec = gpr_ts.tv_nsec; - return linux_ts; -} - /* Convert a timespec to milliseconds: - Very small or negative poll times are clamped to zero to do a non-blocking poll (which becomes spin polling) @@ -1469,200 +1364,35 @@ static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx, return false; } -/* NOTE: This function may modify 'now' */ -static bool acquire_polling_lease(grpc_pollset_worker *worker, - polling_island *pi, gpr_timespec deadline, - gpr_timespec *now) { - bool is_lease_acquired = false; - - gpr_mu_lock(&pi->worker_list_mu); // LOCK - long num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - - if (num_pollers >= g_max_pollers_per_pi) { - push_back_worker_node(&pi->worker_list_head, &worker->pi_list_link); - gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK - - bool is_timeout = false; - int ret; - int timeout_ms = poll_deadline_to_millis_timeout(deadline, *now); - if (timeout_ms == -1) { - ret = sigwaitinfo(&g_wakeup_sig_set, NULL); - } else { - struct timespec sigwait_timeout = millis_to_timespec(timeout_ms); - GRPC_SCHEDULING_START_BLOCKING_REGION; - ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout); - GRPC_SCHEDULING_END_BLOCKING_REGION; - } - - if (ret == -1) { - if (errno == EAGAIN) { - is_timeout = true; - } else { - /* NOTE: This should not happen. If we see these log messages, it means - we are most likely doing something incorrect in the setup * needed - for sigwaitinfo/sigtimedwait */ - gpr_log(GPR_ERROR, - "sigtimedwait failed with retcode: %d (timeout_ms: %d)", errno, - timeout_ms); - } - } - - /* Did the worker come out of sigtimedwait due to a thread that just - exited epoll and kicking it (in release_polling_lease function). */ - bool is_polling_turn = gpr_atm_acq_load(&worker->is_polling_turn); - - /* Did the worker come out of sigtimedwait due to a thread alerting it that - some completion event was (likely) available in the completion queue */ - bool is_kicked = gpr_atm_no_barrier_load(&worker->is_kicked); - - if (is_kicked || is_timeout) { - *now = deadline; /* Essentially make the epoll timeout = 0 */ - } else if (is_polling_turn) { - *now = gpr_now(GPR_CLOCK_MONOTONIC); /* Reduce the epoll timeout */ - } - - gpr_mu_lock(&pi->worker_list_mu); // LOCK - /* The node might have already been removed from the list by the poller - that kicked this. However it is safe to call 'remove_worker_node' on - an already detached node */ - remove_worker_node(&worker->pi_list_link); - /* It is important to read the num_pollers again under the lock so that we - * have the latest num_pollers value that doesn't change while we are doing - * the "(num_pollers < g_max_pollers_per_pi)" a a few lines below */ - num_pollers = gpr_atm_no_barrier_load(&pi->poller_count); - } - - if (num_pollers < g_max_pollers_per_pi) { - gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); - is_lease_acquired = true; - } - - gpr_mu_unlock(&pi->worker_list_mu); // UNLOCK - return is_lease_acquired; -} - -static void release_polling_lease(polling_island *pi, grpc_error **error) { - gpr_mu_lock(&pi->worker_list_mu); - - gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); - worker_node *node = pop_front_worker_node(&pi->worker_list_head); - if (node != NULL) { - grpc_pollset_worker *next_worker = WORKER_FROM_WORKER_LIST_NODE(node); - append_error(error, poller_kick(next_worker), "poller kick error"); - } - - gpr_mu_unlock(&pi->worker_list_mu); -} - #define GRPC_EPOLL_MAX_EVENTS 100 -static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd, - grpc_pollset *pollset, polling_island *pi, - grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline, - sigset_t *sig_mask, grpc_error **error) { - /* Only g_max_pollers_per_pi threads can be doing polling in parallel. - If we cannot get a lease, we cannot continue to do epoll_pwait() */ - if (!acquire_polling_lease(worker, pi, deadline, &now)) { - return; - } - - struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; - int ep_rv; - char *err_msg; - const char *err_desc = "pollset_work_and_unlock"; - - /* timeout_ms is the time between 'now' and 'deadline' */ - int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); - - GRPC_SCHEDULING_START_BLOCKING_REGION; - ep_rv = - epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask); - GRPC_SCHEDULING_END_BLOCKING_REGION; - - /* Give back the lease right away so that some other thread can enter */ - release_polling_lease(pi, error); - - if (ep_rv < 0) { - if (errno != EINTR) { - gpr_asprintf(&err_msg, - "epoll_wait() epoll fd: %d failed with error: %d (%s)", - epoll_fd, errno, strerror(errno)); - append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); - } else { - /* We were interrupted. Save an interation by doing a zero timeout - epoll_wait to see if there are any other events of interest */ - GRPC_POLLING_TRACE("pollset_work: pollset: %p, worker: %p received kick", - (void *)pollset, (void *)worker); - ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); - } - } - -#ifdef GRPC_TSAN - /* See the definition of g_poll_sync for more details */ - gpr_atm_acq_load(&g_epoll_sync); -#endif /* defined(GRPC_TSAN) */ - - for (int i = 0; i < ep_rv; ++i) { - void *data_ptr = ep_ev[i].data.ptr; - if (data_ptr == &global_wakeup_fd) { - grpc_timer_consume_kick(); - append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), - err_desc); - } else if (data_ptr == &pi->workqueue_wakeup_fd) { - append_error(error, - grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), - err_desc); - maybe_do_workqueue_work(exec_ctx, pi); - } else if (data_ptr == &polling_island_wakeup_fd) { - GRPC_POLLING_TRACE( - "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " - "%d) got merged", - (void *)pollset, (void *)worker, epoll_fd); - /* This means that our polling island is merged with a different - island. We do not have to do anything here since the subsequent call - to the function pollset_work_and_unlock() will pick up the correct - epoll_fd */ - } else { - grpc_fd *fd = data_ptr; - int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write_ev = ep_ev[i].events & EPOLLOUT; - if (read_ev || cancel) { - fd_become_readable(exec_ctx, fd, pollset); - } - if (write_ev || cancel) { - fd_become_writable(exec_ctx, fd); - } - } - } -} - /* Note: sig_mask contains the signal mask to use *during* epoll_wait() */ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, - grpc_pollset_worker *worker, - gpr_timespec now, gpr_timespec deadline, + grpc_pollset_worker *worker, int timeout_ms, sigset_t *sig_mask, grpc_error **error) { + struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS]; int epoll_fd = -1; + int ep_rv; polling_island *pi = NULL; + char *err_msg; + const char *err_desc = "pollset_work_and_unlock"; GPR_TIMER_BEGIN("pollset_work_and_unlock", 0); /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the latest polling island pointed by pollset->po.pi - Since epoll_fd is immutable, it is safe to read it without a lock on the - polling island. There is however a possibility that the polling island from - which we got the epoll_fd, got merged with another island in the meantime. - This is okay because in such a case, we will wakeup right-away from - epoll_pwait() (because any merge will poison the old polling island's epoll - set 'polling_island_wakeup_fd') and then pick up the latest polling_island - the next time this function - pollset_work_and_unlock()) is called */ + Since epoll_fd is immutable, we can read it without obtaining the polling + island lock. There is however a possibility that the polling island (from + which we got the epoll_fd) got merged with another island while we are + in this function. This is still okay because in such a case, we will wakeup + right-away from epoll_wait() and pick up the latest polling_island the next + this function (i.e pollset_work_and_unlock()) is called */ if (pollset->po.pi == NULL) { pollset->po.pi = polling_island_create(exec_ctx, NULL, error); if (pollset->po.pi == NULL) { GPR_TIMER_END("pollset_work_and_unlock", 0); - return; /* Fatal error. Cannot continue */ + return; /* Fatal error. We cannot continue */ } PI_ADD_REF(pollset->po.pi, "ps"); @@ -1693,10 +1423,70 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx, the completion queue, so there's no need to poll... so we skip that and redo the complete loop to verify */ if (!maybe_do_workqueue_work(exec_ctx, pi)) { + gpr_atm_no_barrier_fetch_add(&pi->poller_count, 1); g_current_thread_polling_island = pi; - pollset_do_epoll_pwait(exec_ctx, epoll_fd, pollset, pi, worker, now, - deadline, sig_mask, error); + + GRPC_SCHEDULING_START_BLOCKING_REGION; + ep_rv = epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, + sig_mask); + GRPC_SCHEDULING_END_BLOCKING_REGION; + if (ep_rv < 0) { + if (errno != EINTR) { + gpr_asprintf(&err_msg, + "epoll_wait() epoll fd: %d failed with error: %d (%s)", + epoll_fd, errno, strerror(errno)); + append_error(error, GRPC_OS_ERROR(errno, err_msg), err_desc); + } else { + /* We were interrupted. Save an interation by doing a zero timeout + epoll_wait to see if there are any other events of interest */ + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p received kick", + (void *)pollset, (void *)worker); + ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0); + } + } + +#ifdef GRPC_TSAN + /* See the definition of g_poll_sync for more details */ + gpr_atm_acq_load(&g_epoll_sync); +#endif /* defined(GRPC_TSAN) */ + + for (int i = 0; i < ep_rv; ++i) { + void *data_ptr = ep_ev[i].data.ptr; + if (data_ptr == &global_wakeup_fd) { + grpc_timer_consume_kick(); + append_error(error, grpc_wakeup_fd_consume_wakeup(&global_wakeup_fd), + err_desc); + } else if (data_ptr == &pi->workqueue_wakeup_fd) { + append_error(error, + grpc_wakeup_fd_consume_wakeup(&pi->workqueue_wakeup_fd), + err_desc); + maybe_do_workqueue_work(exec_ctx, pi); + } else if (data_ptr == &polling_island_wakeup_fd) { + GRPC_POLLING_TRACE( + "pollset_work: pollset: %p, worker: %p polling island (epoll_fd: " + "%d) got merged", + (void *)pollset, (void *)worker, epoll_fd); + /* This means that our polling island is merged with a different + island. We do not have to do anything here since the subsequent call + to the function pollset_work_and_unlock() will pick up the correct + epoll_fd */ + } else { + grpc_fd *fd = data_ptr; + int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { + fd_become_readable(exec_ctx, fd, pollset); + } + if (write_ev || cancel) { + fd_become_writable(exec_ctx, fd); + } + } + } + g_current_thread_polling_island = NULL; + gpr_atm_no_barrier_fetch_add(&pi->poller_count, -1); } GPR_ASSERT(pi != NULL); @@ -1720,9 +1510,14 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_timespec now, gpr_timespec deadline) { GPR_TIMER_BEGIN("pollset_work", 0); grpc_error *error = GRPC_ERROR_NONE; + int timeout_ms = poll_deadline_to_millis_timeout(deadline, now); + + sigset_t new_mask; grpc_pollset_worker worker; - pollset_worker_init(&worker); + worker.next = worker.prev = NULL; + worker.pt_id = pthread_self(); + gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); if (worker_hdl) *worker_hdl = &worker; @@ -1756,9 +1551,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, misses acting on a kick */ if (!g_initialized_sigmask) { - sigemptyset(&g_wakeup_sig_set); - sigaddset(&g_wakeup_sig_set, grpc_wakeup_signal); - pthread_sigmask(SIG_BLOCK, &g_wakeup_sig_set, &g_orig_sigmask); + sigemptyset(&new_mask); + sigaddset(&new_mask, grpc_wakeup_signal); + pthread_sigmask(SIG_BLOCK, &new_mask, &g_orig_sigmask); sigdelset(&g_orig_sigmask, grpc_wakeup_signal); g_initialized_sigmask = true; /* new_mask: The new thread mask which blocks 'grpc_wakeup_signal'. @@ -1773,7 +1568,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, push_front_worker(pollset, &worker); /* Add worker to pollset */ - pollset_work_and_unlock(exec_ctx, pollset, &worker, now, deadline, + pollset_work_and_unlock(exec_ctx, pollset, &worker, timeout_ms, &g_orig_sigmask, &error); grpc_exec_ctx_flush(exec_ctx); @@ -2126,24 +1921,6 @@ static bool is_epoll_available() { return true; } -/* This is mainly for testing purposes. Checks to see if environment variable - * GRPC_MAX_POLLERS_PER_PI is set and if so, assigns that value to - * g_max_pollers_per_pi (any negative value is considered INT_MAX) */ -static void set_max_pollers_per_island() { - char *s = gpr_getenv("GRPC_MAX_POLLERS_PER_PI"); - if (s) { - g_max_pollers_per_pi = (int)strtol(s, NULL, 10); - if (g_max_pollers_per_pi < 0) { - g_max_pollers_per_pi = INT_MAX; - } - } else { - g_max_pollers_per_pi = INT_MAX; - } - - gpr_log(GPR_INFO, "Max number of pollers per polling island: %d", - g_max_pollers_per_pi); -} - const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { /* If use of signals is disabled, we cannot use epoll engine*/ if (is_grpc_wakeup_signal_initialized && grpc_wakeup_signal < 0) { @@ -2162,8 +1939,6 @@ const grpc_event_engine_vtable *grpc_init_epoll_linux(void) { grpc_use_signal(SIGRTMIN + 6); } - set_max_pollers_per_island(); - fd_global_init(); if (!GRPC_LOG_IF_ERROR("pollset_global_init", pollset_global_init())) { diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index 13409a4de83..07a9ed570a1 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -44,6 +44,7 @@ #include #include +#include "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h" #include "src/core/lib/iomgr/ev_epoll_linux.h" #include "src/core/lib/iomgr/ev_poll_posix.h" #include "src/core/lib/support/env.h" @@ -68,6 +69,7 @@ static const event_engine_factory g_factories[] = { {"epoll", grpc_init_epoll_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, + {"epoll-limited", grpc_init_epoll_limited_pollers_linux}, }; static void add(const char *beg, const char *end, char ***ss, size_t *ns) { diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 42c13b2cb63..c3053612e48 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -97,6 +97,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/iomgr/endpoint_pair_uv.c', 'src/core/lib/iomgr/endpoint_pair_windows.c', 'src/core/lib/iomgr/error.c', + 'src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c', 'src/core/lib/iomgr/ev_epoll_linux.c', 'src/core/lib/iomgr/ev_poll_posix.c', 'src/core/lib/iomgr/ev_posix.c', diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 9664234f9f4..5ce89e7a8e1 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -939,6 +939,8 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 51d77c2b521..54a7ed4e971 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1078,6 +1078,8 @@ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c \ +src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h \ src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_epoll_linux.h \ src/core/lib/iomgr/ev_poll_posix.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index c9628234379..f2c467e4ea1 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7750,6 +7750,7 @@ "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", + "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h", "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", @@ -7891,6 +7892,8 @@ "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", + "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c", + "src/core/lib/iomgr/ev_epoll_limited_pollers_linux.h", "src/core/lib/iomgr/ev_epoll_linux.c", "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_poll_posix.c", diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index 32d2e09a588..4d44d450a21 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -396,6 +396,7 @@ + @@ -610,6 +611,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index a3346bc2979..b2afed1e504 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -184,6 +184,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -908,6 +911,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 28ccefc651c..fd207a2c794 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -390,6 +390,7 @@ + @@ -594,6 +595,8 @@ + + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 83f869dab36..531ae520474 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -169,6 +169,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -875,6 +878,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 86b5856d68a..9e0fb5de8d4 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -320,6 +320,7 @@ + @@ -551,6 +552,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 943ad521f75..4abf85b10f0 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -64,6 +64,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -875,6 +878,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index df89932a97c..60deba9243f 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -215,6 +215,7 @@ + @@ -383,6 +384,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 22cfbe14d4d..44a29fadb77 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -121,6 +121,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -623,6 +626,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 2db0ffa6920..6fdf6bb6903 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -310,6 +310,7 @@ + @@ -518,6 +519,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index c7d6670db1b..707ab9e098e 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -67,6 +67,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr @@ -785,6 +788,9 @@ src\core\lib\iomgr + + src\core\lib\iomgr + src\core\lib\iomgr From f070d4158c2dbc32d3d27594a7be63c4582141bd Mon Sep 17 00:00:00 2001 From: yang-g Date: Mon, 8 May 2017 15:02:28 -0700 Subject: [PATCH 138/177] Lower the level of a log message to INFO --- src/cpp/server/server_cc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index b4d6961db89..2f89aa3dce1 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -521,7 +521,7 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { if (health_check_service_ == nullptr && !health_check_service_disabled_ && DefaultHealthCheckServiceEnabled()) { if (sync_server_cqs_->empty()) { - gpr_log(GPR_ERROR, + gpr_log(GPR_INFO, "Default health check service disabled at async-only server."); } else { auto* default_hc_service = new DefaultHealthCheckService; From b6ac9b4573d5f8bc9fbac09482d254b6a7a241b2 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 8 May 2017 17:10:31 -0700 Subject: [PATCH 139/177] Add unbalanced unary benchmark --- .../microbenchmarks/bm_fullstack_trickle.cc | 106 ++++++++++++++++-- 1 file changed, 96 insertions(+), 10 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index fc99b06dbb2..b0f7b34bb50 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -77,13 +77,17 @@ static void write_csv(std::ostream* out, A0&& a0, Arg&&... arg) { class TrickledCHTTP2 : public EndpointPairFixture { public: - TrickledCHTTP2(Service* service, size_t message_size, + TrickledCHTTP2(Service* service, bool streaming, size_t req_size, size_t resp_size, size_t kilobits_per_second) : EndpointPairFixture(service, MakeEndpoints(kilobits_per_second), FixtureConfiguration()) { if (FLAGS_log) { std::ostringstream fn; - fn << "trickle." << message_size << "." << kilobits_per_second << ".csv"; + fn << "trickle." << (streaming ? "streaming" + : "unary" ) + << "." << req_size << "." << resp_size + << "." << kilobits_per_second + << ".csv"; log_.reset(new std::ofstream(fn.str().c_str())); write_csv(log_.get(), "t", "iteration", "client_backlog", "server_backlog", "client_t_stall", "client_s_stall", @@ -242,8 +246,9 @@ static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok, static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { EchoTestService::AsyncService service; - std::unique_ptr fixture( - new TrickledCHTTP2(&service, state.range(0), state.range(1))); + std::unique_ptr fixture(new TrickledCHTTP2( + &service, true, state.range(0) /* req_size */, + state.range(0) /* resp_size */, state.range(1) /* bw in kbit/s */)); { EchoResponse send_response; EchoResponse recv_response; @@ -314,11 +319,7 @@ static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) { state.SetBytesProcessed(state.range(0) * state.iterations()); } -/******************************************************************************* - * CONFIGURATIONS - */ - -static void TrickleArgs(benchmark::internal::Benchmark* b) { +static void StreamingTrickleArgs(benchmark::internal::Benchmark* b) { for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) { for (int j = 64; j <= 128 * 1024 * 1024; j *= 8) { double expected_time = @@ -328,8 +329,93 @@ static void TrickleArgs(benchmark::internal::Benchmark* b) { } } } +BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(StreamingTrickleArgs); -BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs); +static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { + EchoTestService::AsyncService service; + std::unique_ptr fixture(new TrickledCHTTP2( + &service, true, state.range(0) /* req_size */, + state.range(1) /* resp_size */, state.range(2) /* bw in kbit/s */)); + EchoRequest send_request; + EchoResponse send_response; + EchoResponse recv_response; + if (state.range(0) > 0) { + send_request.set_message(std::string(state.range(0), 'a')); + } + if (state.range(1) > 0) { + send_response.set_message(std::string(state.range(1), 'a')); + } + Status recv_status; + struct ServerEnv { + ServerContext ctx; + EchoRequest recv_request; + grpc::ServerAsyncResponseWriter response_writer; + ServerEnv() : response_writer(&ctx) {} + }; + uint8_t server_env_buffer[2 * sizeof(ServerEnv)]; + ServerEnv* server_env[2] = { + reinterpret_cast(server_env_buffer), + reinterpret_cast(server_env_buffer + sizeof(ServerEnv))}; + new (server_env[0]) ServerEnv; + new (server_env[1]) ServerEnv; + service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request, + &server_env[0]->response_writer, fixture->cq(), + fixture->cq(), tag(0)); + service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request, + &server_env[1]->response_writer, fixture->cq(), + fixture->cq(), tag(1)); + std::unique_ptr stub( + EchoTestService::NewStub(fixture->channel())); + while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); + recv_response.Clear(); + ClientContext cli_ctx; + std::unique_ptr> response_reader( + stub->AsyncEcho(&cli_ctx, send_request, fixture->cq())); + void* t; + bool ok; + TrickleCQNext(fixture.get(), &t, &ok, state.iterations()); + GPR_ASSERT(ok); + GPR_ASSERT(t == tag(0) || t == tag(1)); + intptr_t slot = reinterpret_cast(t); + ServerEnv* senv = server_env[slot]; + senv->response_writer.Finish(send_response, Status::OK, tag(3)); + response_reader->Finish(&recv_response, &recv_status, tag(4)); + for (int i = (1 << 3) | (1 << 4); i != 0;) { + TrickleCQNext(fixture.get(), &t, &ok, state.iterations()); + GPR_ASSERT(ok); + int tagnum = (int)reinterpret_cast(t); + GPR_ASSERT(i & (1 << tagnum)); + i -= 1 << tagnum; + } + GPR_ASSERT(recv_status.ok()); + + senv->~ServerEnv(); + senv = new (senv) ServerEnv(); + service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, + fixture->cq(), fixture->cq(), tag(slot)); + } + fixture->Finish(state); + fixture.reset(); + server_env[0]->~ServerEnv(); + server_env[1]->~ServerEnv(); + state.SetBytesProcessed(state.range(0) * state.iterations() + + state.range(1) * state.iterations()); +} + +static void UnaryTrickleArgs(benchmark::internal::Benchmark* b) { + for (int i = 1; i <= 128 * 1024 * 1024; i *= 32) { + for (int j = 1; j <= 128 * 1024 * 1024; j *= 32) { + for (int k = 64; k <= 128 * 1024 * 1024; k *= 16) { + double expected_time = + static_cast(14 + i + k) / (125.0 * 2 * static_cast(j)); + if (expected_time > 2.0) continue; + b->Args({i, j, k}); + } + } + } +} +BENCHMARK(BM_PumpUnbalancedUnary_Trickle)->Apply(UnaryTrickleArgs); } } From ffac55dfd634a62d54c8f152bbbce2bb51e7bc31 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 5 May 2017 13:54:03 -0700 Subject: [PATCH 140/177] Refactor client logic into superclass with generic methods, improve documentation --- src/node/index.js | 11 +- src/node/src/client.js | 790 ++++++++++++++++++++++----------------- src/node/src/common.js | 67 +++- src/node/src/metadata.js | 13 +- 4 files changed, 525 insertions(+), 356 deletions(-) diff --git a/src/node/index.js b/src/node/index.js index 071bfd7927b..76ab1744b00 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -31,6 +31,10 @@ * */ +/** + * @module + */ + 'use strict'; var path = require('path'); @@ -256,5 +260,10 @@ exports.getClientChannel = client.getClientChannel; exports.waitForClientReady = client.waitForClientReady; exports.closeClient = function closeClient(client_obj) { - client.getClientChannel(client_obj).close(); + client.Client.prototype.close.apply(client_obj); }; + +/** + * @see module:src/client.Client + */ +exports.Client = client.Client; diff --git a/src/node/src/client.js b/src/node/src/client.js index 1aaf35c16cb..75567c69ecd 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -37,7 +37,7 @@ * This module contains the factory method for creating Client classes, and the * method calling code for all types of methods. * - * For example, to create a client and call a method on it: + * @example Create a client and call a method on it * * var proto_obj = grpc.load(proto_file_path); * var Client = proto_obj.package.subpackage.ServiceName; @@ -68,14 +68,33 @@ var Duplex = stream.Duplex; var util = require('util'); var version = require('../../../package.json').version; +util.inherits(ClientUnaryCall, EventEmitter); + +/** + * An EventEmitter. Used for unary calls + * @constructor + * @extends external:EventEmitter + * @param {grpc.Call} call The call object associated with the request + */ +function ClientUnaryCall(call) { + EventEmitter.call(this); + this.call = call; +} + util.inherits(ClientWritableStream, Writable); /** * A stream that the client can write to. Used for calls that are streaming from * the client side. * @constructor + * @extends external:Writable + * @borrows module:src/client~ClientUnaryCall#cancel as + * module:src/client~ClientWritableStream#cancel + * @borrows module:src/client~ClientUnaryCall#getPeer as + * module:src/client~ClientWritableStream#getPeer * @param {grpc.Call} call The call object to send data with - * @param {function(*):Buffer=} serialize Serialization function for writes. + * @param {module:src/common~serialize=} [serialize=identity] Serialization + * function for writes. */ function ClientWritableStream(call, serialize) { Writable.call(this, {objectMode: true}); @@ -134,8 +153,14 @@ util.inherits(ClientReadableStream, Readable); * A stream that the client can read from. Used for calls that are streaming * from the server side. * @constructor + * @extends external:Readable + * @borrows module:src/client~ClientUnaryCall#cancel as + * module:src/client~ClientReadableStream#cancel + * @borrows module:src/client~ClientUnaryCall#getPeer as + * module:src/client~ClientReadableStream#getPeer * @param {grpc.Call} call The call object to read data with - * @param {function(Buffer):*=} deserialize Deserialization function for reads + * @param {module:src/common~deserialize=} [deserialize=identity] + * Deserialization function for reads */ function ClientReadableStream(call, deserialize) { Readable.call(this, {objectMode: true}); @@ -155,6 +180,7 @@ function ClientReadableStream(call, deserialize) { * parameter indicates that the call should end with that status. status * defaults to OK if not provided. * @param {Object!} status The status that the call should end with + * @access private */ function _readsDone(status) { /* jshint validthis: true */ @@ -173,6 +199,7 @@ ClientReadableStream.prototype._readsDone = _readsDone; /** * Called to indicate that we have received a status from the server. + * @access private */ function _receiveStatus(status) { /* jshint validthis: true */ @@ -185,6 +212,7 @@ ClientReadableStream.prototype._receiveStatus = _receiveStatus; /** * If we have both processed all incoming messages and received the status from * the server, emit the status. Otherwise, do nothing. + * @access private */ function _emitStatusIfDone() { /* jshint validthis: true */ @@ -270,10 +298,16 @@ util.inherits(ClientDuplexStream, Duplex); * A stream that the client can read from or write to. Used for calls with * duplex streaming. * @constructor + * @extends external:Duplex + * @borrows module:src/client~ClientUnaryCall#cancel as + * module:src/client~ClientDuplexStream#cancel + * @borrows module:src/client~ClientUnaryCall#getPeer as + * module:src/client~ClientDuplexStream#getPeer * @param {grpc.Call} call Call object to proxy - * @param {function(*):Buffer=} serialize Serialization function for requests - * @param {function(Buffer):*=} deserialize Deserialization function for - * responses + * @param {module:src/common~serialize=} [serialize=identity] Serialization + * function for requests + * @param {module:src/common~deserialize=} [deserialize=identity] + * Deserialization function for responses */ function ClientDuplexStream(call, serialize, deserialize) { Duplex.call(this, {objectMode: true}); @@ -300,12 +334,14 @@ ClientDuplexStream.prototype._write = _write; /** * Cancel the ongoing call + * @alias module:src/client~ClientUnaryCall#cancel */ function cancel() { /* jshint validthis: true */ this.call.cancel(); } +ClientUnaryCall.prototype.cancel = cancel; ClientReadableStream.prototype.cancel = cancel; ClientWritableStream.prototype.cancel = cancel; ClientDuplexStream.prototype.cancel = cancel; @@ -313,21 +349,49 @@ ClientDuplexStream.prototype.cancel = cancel; /** * Get the endpoint this call/stream is connected to. * @return {string} The URI of the endpoint + * @alias module:src/client~ClientUnaryCall#getPeer */ function getPeer() { /* jshint validthis: true */ return this.call.getPeer(); } +ClientUnaryCall.prototype.getPeer = getPeer; ClientReadableStream.prototype.getPeer = getPeer; ClientWritableStream.prototype.getPeer = getPeer; ClientDuplexStream.prototype.getPeer = getPeer; +/** + * Any client call type + * @typedef {(ClientUnaryCall|ClientReadableStream| + * ClientWritableStream|ClientDuplexStream)} + * module:src/client~Call + */ + +/** + * Options that can be set on a call. + * @typedef {Object} module:src/client~CallOptions + * @property {(date|number)} deadline The deadline for the entire call to + * complete. A value of Infinity indicates that no deadline should be set. + * @property {(string)} host Server hostname to set on the call. Only meaningful + * if different from the server address used to construct the client. + * @property {module:src/client~Call} parent Parent call. Used in servers when + * making a call as part of the process of handling a call. Used to + * propagate some information automatically, as specified by + * propagate_flags. + * @property {number} propagate_flags Indicates which properties of a parent + * call should propagate to this call. Bitwise combination of flags in + * [grpc.propagate]{@link module:index.propagate}. + * @property {module:src/credentials~CallCredentials} credentials The + * credentials that should be used to make this particular call. + */ + /** * Get a call object built with the provided options. Keys for options are * 'deadline', which takes a date or number, and 'host', which takes a string * and overrides the hostname to connect to. - * @param {Object} options Options map. + * @access private + * @param {module:src/client~CallOptions=} options Options object. */ function getCall(channel, method, options) { var deadline; @@ -354,315 +418,380 @@ function getCall(channel, method, options) { } /** - * Get a function that can make unary requests to the specified method. - * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeUnaryRequest + * A generic gRPC client. Primarily useful as a base class for generated clients + * @alias module:src/client.Client + * @constructor + * @param {string} address Server address to connect to + * @param {module:src/credentials~ChannelCredentials} credentials Credentials to + * use to connect to the server + * @param {Object} options Options to apply to channel creation */ -function makeUnaryRequestFunction(method, serialize, deserialize) { - /** - * Make a unary request with this method on the given channel with the given - * argument, callback, etc. - * @this {Client} Client object. Must have a channel member. - * @param {*} argument The argument to the call. Should be serializable with - * serialize - * @param {Metadata=} metadata Metadata to add to the call - * @param {Object=} options Options map - * @param {function(?Error, value=)} callback The callback to for when the - * response is received - * @return {EventEmitter} An event emitter for stream related events +var Client = exports.Client = function Client(address, credentials, options) { + if (!options) { + options = {}; + } + /* Append the grpc-node user agent string after the application user agent + * string, and put the combination at the beginning of the user agent string */ - function makeUnaryRequest(argument, metadata, options, callback) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. This allows for simple handling of optional arguments in the - * middle of the argument list, and also provides type checking. */ - var args = arguejs({argument: null, metadata: [Metadata, new Metadata()], - options: [Object], callback: Function}, arguments); - var emitter = new EventEmitter(); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - emitter.cancel = function cancel() { - call.cancel(); - }; - emitter.getPeer = function getPeer() { - return call.getPeer(); - }; - var client_batch = {}; - var message = serialize(args.argument); - if (args.options) { - message.grpcWriteFlags = args.options.flags; - } - - client_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - client_batch[grpc.opType.SEND_MESSAGE] = message; - client_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; - client_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - client_batch[grpc.opType.RECV_MESSAGE] = true; - client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(client_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - var status = response.status; - var error; - var deserialized; - emitter.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - if (status.code === grpc.status.OK) { - if (err) { - // Got a batch error, but OK status. Something went wrong - args.callback(err); - return; - } else { - try { - deserialized = deserialize(response.read); - } catch (e) { - /* Change status to indicate bad server response. This will result - * in passing an error to the callback */ - status = { - code: grpc.status.INTERNAL, - details: 'Failed to parse server response' - }; - } - } - } - if (status.code !== grpc.status.OK) { - error = new Error(status.details); - error.code = status.code; - error.metadata = status.metadata; - args.callback(error); - } else { - args.callback(null, deserialized); - } - emitter.emit('status', status); - }); - return emitter; + if (options['grpc.primary_user_agent']) { + options['grpc.primary_user_agent'] += ' '; + } else { + options['grpc.primary_user_agent'] = ''; } - return makeUnaryRequest; -} + options['grpc.primary_user_agent'] += 'grpc-node/' + version; + /* Private fields use $ as a prefix instead of _ because it is an invalid + * prefix of a method name */ + this.$channel = new grpc.Channel(address, credentials, options); +}; /** - * Get a function that can make client stream requests to the specified method. + * @typedef {Error} module:src/client.Client~ServiceError + * @property {number} code The error code, a key of + * [grpc.status]{@link module:src/client.status} + * @property {module:metadata.Metadata} metadata Metadata sent with the status + * by the server, if any + */ + +/** + * @callback module:src/client.Client~requestCallback + * @param {?module:src/client.Client~ServiceError} error The error, if the call + * failed + * @param {*} value The response value, if the call succeeded + */ + +/** + * Make a unary request to the given method, using the given serialize + * and deserialize functions, with the given argument. * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeClientStreamRequest + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {*} argument The argument to the call. Should be serializable with + * serialize + * @param {module:src/metadata.Metadata=} metadata Metadata to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @param {module:src/client.Client~requestCallback} callback The callback to + * for when the response is received + * @return {EventEmitter} An event emitter for stream related events */ -function makeClientStreamRequestFunction(method, serialize, deserialize) { - /** - * Make a client stream request with this method on the given channel with the - * given callback, etc. - * @this {Client} Client object. Must have a channel member. - * @param {Metadata=} metadata Array of metadata key/value pairs to add to the - * call - * @param {Object=} options Options map - * @param {function(?Error, value=)} callback The callback to for when the - * response is received - * @return {EventEmitter} An event emitter for stream related events - */ - function makeClientStreamRequest(metadata, options, callback) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. This allows for simple handling of optional arguments in the - * middle of the argument list, and also provides type checking. */ - var args = arguejs({metadata: [Metadata, new Metadata()], - options: [Object], callback: Function}, arguments); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - var stream = new ClientWritableStream(call, serialize); - var metadata_batch = {}; - metadata_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - metadata_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - call.startBatch(metadata_batch, function(err, response) { +Client.prototype.makeUnaryRequest = function(method, serialize, deserialize, + argument, metadata, options, + callback) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. This allows for simple handling of optional arguments in the + * middle of the argument list, and also provides type checking. */ + var args = arguejs({method: String, serialize: Function, + deserialize: Function, + argument: null, metadata: [Metadata, new Metadata()], + options: [Object], callback: Function}, arguments); + var call = getCall(this.$channel, method, args.options); + var emitter = new ClientUnaryCall(call); + metadata = args.metadata.clone(); + var client_batch = {}; + var message = serialize(args.argument); + if (args.options) { + message.grpcWriteFlags = args.options.flags; + } + + client_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + client_batch[grpc.opType.SEND_MESSAGE] = message; + client_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; + client_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + client_batch[grpc.opType.RECV_MESSAGE] = true; + client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(client_batch, function(err, response) { + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + var status = response.status; + var error; + var deserialized; + emitter.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + if (status.code === grpc.status.OK) { if (err) { - // The call has stopped for some reason. A non-OK status will arrive - // in the other batch. + // Got a batch error, but OK status. Something went wrong + args.callback(err); return; - } - stream.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - }); - var client_batch = {}; - client_batch[grpc.opType.RECV_MESSAGE] = true; - client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(client_batch, function(err, response) { - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - var status = response.status; - var error; - var deserialized; - if (status.code === grpc.status.OK) { - if (err) { - // Got a batch error, but OK status. Something went wrong - args.callback(err); - return; - } else { - try { - deserialized = deserialize(response.read); - } catch (e) { - /* Change status to indicate bad server response. This will result - * in passing an error to the callback */ - status = { - code: grpc.status.INTERNAL, - details: 'Failed to parse server response' - }; - } - } - } - if (status.code !== grpc.status.OK) { - error = new Error(response.status.details); - error.code = status.code; - error.metadata = status.metadata; - args.callback(error); } else { - args.callback(null, deserialized); + try { + deserialized = deserialize(response.read); + } catch (e) { + /* Change status to indicate bad server response. This will result + * in passing an error to the callback */ + status = { + code: grpc.status.INTERNAL, + details: 'Failed to parse server response' + }; + } } - stream.emit('status', status); - }); - return stream; - } - return makeClientStreamRequest; -} + } + if (status.code !== grpc.status.OK) { + error = new Error(status.details); + error.code = status.code; + error.metadata = status.metadata; + args.callback(error); + } else { + args.callback(null, deserialized); + } + emitter.emit('status', status); + }); + return emitter; +}; /** - * Get a function that can make server stream requests to the specified method. + * Make a client stream request to the given method, using the given serialize + * and deserialize functions, with the given argument. * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeServerStreamRequest + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {module:src/metadata.Metadata=} metadata Array of metadata key/value + * pairs to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @param {Client~requestCallback} callback The callback to for when the + * response is received + * @return {module:src/client~ClientWritableStream} An event emitter for stream + * related events */ -function makeServerStreamRequestFunction(method, serialize, deserialize) { - /** - * Make a server stream request with this method on the given channel with the - * given argument, etc. - * @this {SurfaceClient} Client object. Must have a channel member. - * @param {*} argument The argument to the call. Should be serializable with - * serialize - * @param {Metadata=} metadata Array of metadata key/value pairs to add to the - * call - * @param {Object} options Options map - * @return {EventEmitter} An event emitter for stream related events - */ - function makeServerStreamRequest(argument, metadata, options) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. */ - var args = arguejs({argument: null, metadata: [Metadata, new Metadata()], - options: [Object]}, arguments); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - var stream = new ClientReadableStream(call, deserialize); - var start_batch = {}; - var message = serialize(args.argument); - if (args.options) { - message.grpcWriteFlags = args.options.flags; +Client.prototype.makeClientStreamRequest = function(method, serialize, + deserialize, metadata, + options, callback) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. This allows for simple handling of optional arguments in the + * middle of the argument list, and also provides type checking. */ + var args = arguejs({method:String, serialize: Function, + deserialize: Function, + metadata: [Metadata, new Metadata()], + options: [Object], callback: Function}, arguments); + var call = getCall(this.$channel, method, args.options); + metadata = args.metadata.clone(); + var stream = new ClientWritableStream(call, serialize); + var metadata_batch = {}; + metadata_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + metadata_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + call.startBatch(metadata_batch, function(err, response) { + if (err) { + // The call has stopped for some reason. A non-OK status will arrive + // in the other batch. + return; } - start_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - start_batch[grpc.opType.SEND_MESSAGE] = message; - start_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; - call.startBatch(start_batch, function(err, response) { - if (err) { - // The call has stopped for some reason. A non-OK status will arrive - // in the other batch. - return; - } - stream.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - }); - var status_batch = {}; - status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(status_batch, function(err, response) { + stream.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + }); + var client_batch = {}; + client_batch[grpc.opType.RECV_MESSAGE] = true; + client_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(client_batch, function(err, response) { + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + var status = response.status; + var error; + var deserialized; + if (status.code === grpc.status.OK) { if (err) { - stream.emit('error', err); + // Got a batch error, but OK status. Something went wrong + args.callback(err); return; + } else { + try { + deserialized = deserialize(response.read); + } catch (e) { + /* Change status to indicate bad server response. This will result + * in passing an error to the callback */ + status = { + code: grpc.status.INTERNAL, + details: 'Failed to parse server response' + }; + } } - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream._receiveStatus(response.status); - }); - return stream; - } - return makeServerStreamRequest; -} + } + if (status.code !== grpc.status.OK) { + error = new Error(response.status.details); + error.code = status.code; + error.metadata = status.metadata; + args.callback(error); + } else { + args.callback(null, deserialized); + } + stream.emit('status', status); + }); + return stream; +}; /** - * Get a function that can make bidirectional stream requests to the specified - * method. + * Make a server stream request to the given method, with the given serialize + * and deserialize function, using the given argument * @param {string} method The name of the method to request - * @param {function(*):Buffer} serialize The serialization function for inputs - * @param {function(Buffer)} deserialize The deserialization function for - * outputs - * @return {Function} makeBidiStreamRequest + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {*} argument The argument to the call. Should be serializable with + * serialize + * @param {module:src/metadata.Metadata=} metadata Array of metadata key/value + * pairs to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @return {module:src/client~ClientReadableStream} An event emitter for stream + * related events */ -function makeBidiStreamRequestFunction(method, serialize, deserialize) { - /** - * Make a bidirectional stream request with this method on the given channel. - * @this {SurfaceClient} Client object. Must have a channel member. - * @param {Metadata=} metadata Array of metadata key/value pairs to add to the - * call - * @param {Options} options Options map - * @return {EventEmitter} An event emitter for stream related events - */ - function makeBidiStreamRequest(metadata, options) { - /* jshint validthis: true */ - /* While the arguments are listed in the function signature, those variables - * are not used directly. Instead, ArgueJS processes the arguments - * object. */ - var args = arguejs({metadata: [Metadata, new Metadata()], - options: [Object]}, arguments); - var call = getCall(this.$channel, method, args.options); - metadata = args.metadata.clone(); - var stream = new ClientDuplexStream(call, serialize, deserialize); - var start_batch = {}; - start_batch[grpc.opType.SEND_INITIAL_METADATA] = - metadata._getCoreRepresentation(); - start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; - call.startBatch(start_batch, function(err, response) { - if (err) { - // The call has stopped for some reason. A non-OK status will arrive - // in the other batch. - return; - } - stream.emit('metadata', Metadata._fromCoreRepresentation( - response.metadata)); - }); - var status_batch = {}; - status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; - call.startBatch(status_batch, function(err, response) { - if (err) { - stream.emit('error', err); - return; - } - response.status.metadata = Metadata._fromCoreRepresentation( - response.status.metadata); - stream._receiveStatus(response.status); - }); - return stream; +Client.prototype.makeServerStreamRequest = function(method, serialize, + deserialize, argument, + metadata, options) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. */ + var args = arguejs({method:String, serialize: Function, + deserialize: Function, + argument: null, metadata: [Metadata, new Metadata()], + options: [Object]}, arguments); + var call = getCall(this.$channel, method, args.options); + metadata = args.metadata.clone(); + var stream = new ClientReadableStream(call, deserialize); + var start_batch = {}; + var message = serialize(args.argument); + if (args.options) { + message.grpcWriteFlags = args.options.flags; } - return makeBidiStreamRequest; -} + start_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + start_batch[grpc.opType.SEND_MESSAGE] = message; + start_batch[grpc.opType.SEND_CLOSE_FROM_CLIENT] = true; + call.startBatch(start_batch, function(err, response) { + if (err) { + // The call has stopped for some reason. A non-OK status will arrive + // in the other batch. + return; + } + stream.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + }); + var status_batch = {}; + status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(status_batch, function(err, response) { + if (err) { + stream.emit('error', err); + return; + } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); + }); + return stream; +}; + + +/** + * Make a bidirectional stream request with this method on the given channel. + * @param {string} method The name of the method to request + * @param {module:src/common~serialize} serialize The serialization function for + * inputs + * @param {module:src/common~deserialize} deserialize The deserialization + * function for outputs + * @param {module:src/metadata.Metadata=} metadata Array of metadata key/value + * pairs to add to the call + * @param {module:src/client~CallOptions=} options Options map + * @return {module:src/client~ClientDuplexStream} An event emitter for stream + * related events + */ +Client.prototype.makeBidiStreamRequest = function(method, serialize, + deserialize, metadata, + options) { + /* While the arguments are listed in the function signature, those variables + * are not used directly. Instead, ArgueJS processes the arguments + * object. */ + var args = arguejs({method:String, serialize: Function, + deserialize: Function, + metadata: [Metadata, new Metadata()], + options: [Object]}, arguments); + var call = getCall(this.$channel, method, args.options); + metadata = args.metadata.clone(); + var stream = new ClientDuplexStream(call, serialize, deserialize); + var start_batch = {}; + start_batch[grpc.opType.SEND_INITIAL_METADATA] = + metadata._getCoreRepresentation(); + start_batch[grpc.opType.RECV_INITIAL_METADATA] = true; + call.startBatch(start_batch, function(err, response) { + if (err) { + // The call has stopped for some reason. A non-OK status will arrive + // in the other batch. + return; + } + stream.emit('metadata', Metadata._fromCoreRepresentation( + response.metadata)); + }); + var status_batch = {}; + status_batch[grpc.opType.RECV_STATUS_ON_CLIENT] = true; + call.startBatch(status_batch, function(err, response) { + if (err) { + stream.emit('error', err); + return; + } + response.status.metadata = Metadata._fromCoreRepresentation( + response.status.metadata); + stream._receiveStatus(response.status); + }); + return stream; +}; +Client.prototype.close = function() { + this.$channel.close(); +}; + +/** + * Return the underlying channel object for the specified client + * @return {Channel} The channel + */ +Client.prototype.getChannel = function() { + return this.$channel; +}; + +/** + * Wait for the client to be ready. The callback will be called when the + * client has successfully connected to the server, and it will be called + * with an error if the attempt to connect to the server has unrecoverablly + * failed or if the deadline expires. This function will make the channel + * start connecting if it has not already done so. + * @param {(Date|Number)} deadline When to stop waiting for a connection. Pass + * Infinity to wait forever. + * @param {function(Error)} callback The callback to call when done attempting + * to connect. + */ +Client.prototype.waitForReady = function(deadline, callback) { + var self = this; + var checkState = function(err) { + if (err) { + callback(new Error('Failed to connect before the deadline')); + return; + } + var new_state = self.$channel.getConnectivityState(true); + if (new_state === grpc.connectivityState.READY) { + callback(); + } else if (new_state === grpc.connectivityState.FATAL_FAILURE) { + callback(new Error('Failed to connect to server')); + } else { + self.$channel.watchConnectivityState(new_state, deadline, checkState); + } + }; + checkState(); +}; /** * Map with short names for each of the requester maker functions. Used in * makeClientConstructor + * @access private */ -var requester_makers = { - unary: makeUnaryRequestFunction, - server_stream: makeServerStreamRequestFunction, - client_stream: makeClientStreamRequestFunction, - bidi: makeBidiStreamRequestFunction +var requester_funcs = { + unary: Client.prototype.makeUnaryRequest, + server_stream: Client.prototype.makeServerStreamRequest, + client_stream: Client.prototype.makeClientStreamRequest, + bidi: Client.prototype.makeBidiStreamRequest }; function getDefaultValues(metadata, options) { @@ -675,6 +804,7 @@ function getDefaultValues(metadata, options) { /** * Map with wrappers for each type of requester function to make it use the old * argument order with optional arguments after the callback. + * @access private */ var deprecated_request_wrap = { unary: function(makeUnaryRequest) { @@ -700,55 +830,33 @@ var deprecated_request_wrap = { }; /** - * Creates a constructor for a client with the given methods. The methods object - * maps method name to an object with the following keys: - * path: The path on the server for accessing the method. For example, for - * protocol buffers, we use "/service_name/method_name" - * requestStream: bool indicating whether the client sends a stream - * resonseStream: bool indicating whether the server sends a stream - * requestSerialize: function to serialize request objects - * responseDeserialize: function to deserialize response objects - * @param {Object} methods An object mapping method names to method attributes + * Creates a constructor for a client with the given methods, as specified in + * the methods argument. + * @param {module:src/common~ServiceDefinition} methods An object mapping + * method names to method attributes * @param {string} serviceName The fully qualified name of the service - * @param {Object} class_options An options object. Currently only uses the key - * deprecatedArgumentOrder, a boolean that Indicates that the old argument - * order should be used for methods, with optional arguments at the end - * instead of the callback at the end. Defaults to false. This option is - * only a temporary stopgap measure to smooth an API breakage. + * @param {Object} class_options An options object. + * @param {boolean=} [class_options.deprecatedArgumentOrder=false] Indicates + * that the old argument order should be used for methods, with optional + * arguments at the end instead of the callback at the end. This option + * is only a temporary stopgap measure to smooth an API breakage. * It is deprecated, and new code should not use it. - * @return {function(string, Object)} New client constructor + * @return {function(string, Object)} New client constructor, which is a + * subclass of [grpc.Client]{@link module:src/client.Client}, and has the + * same arguments as that constructor. */ exports.makeClientConstructor = function(methods, serviceName, class_options) { if (!class_options) { class_options = {}; } - /** - * Create a client with the given methods - * @constructor - * @param {string} address The address of the server to connect to - * @param {grpc.Credentials} credentials Credentials to use to connect - * to the server - * @param {Object} options Options to pass to the underlying channel - */ - function Client(address, credentials, options) { - if (!options) { - options = {}; - } - /* Append the grpc-node user agent string after the application user agent - * string, and put the combination at the beginning of the user agent string - */ - if (options['grpc.primary_user_agent']) { - options['grpc.primary_user_agent'] += ' '; - } else { - options['grpc.primary_user_agent'] = ''; - } - options['grpc.primary_user_agent'] += 'grpc-node/' + version; - /* Private fields use $ as a prefix instead of _ because it is an invalid - * prefix of a method name */ - this.$channel = new grpc.Channel(address, credentials, options); + + function ServiceClient(address, credentials, options) { + Client.call(this, address, credentials, options); } + util.inherits(ServiceClient, Client); + _.each(methods, function(attrs, name) { var method_type; if (_.startsWith(name, '$')) { @@ -769,20 +877,20 @@ exports.makeClientConstructor = function(methods, serviceName, } var serialize = attrs.requestSerialize; var deserialize = attrs.responseDeserialize; - var method_func = requester_makers[method_type]( - attrs.path, serialize, deserialize); + var method_func = _.partial(requester_funcs[method_type], attrs.path, + serialize, deserialize); if (class_options.deprecatedArgumentOrder) { - Client.prototype[name] = deprecated_request_wrap(method_func); + ServiceClient.prototype[name] = deprecated_request_wrap(method_func); } else { - Client.prototype[name] = method_func; + ServiceClient.prototype[name] = method_func; } // Associate all provided attributes with the method - _.assign(Client.prototype[name], attrs); + _.assign(ServiceClient.prototype[name], attrs); }); - Client.service = methods; + ServiceClient.service = methods; - return Client; + return ServiceClient; }; /** @@ -791,7 +899,7 @@ exports.makeClientConstructor = function(methods, serviceName, * @return {Channel} The channel */ exports.getClientChannel = function(client) { - return client.$channel; + return Client.prototype.getChannel.apply(client); }; /** @@ -807,21 +915,7 @@ exports.getClientChannel = function(client) { * to connect. */ exports.waitForClientReady = function(client, deadline, callback) { - var checkState = function(err) { - if (err) { - callback(new Error('Failed to connect before the deadline')); - return; - } - var new_state = client.$channel.getConnectivityState(true); - if (new_state === grpc.connectivityState.READY) { - callback(); - } else if (new_state === grpc.connectivityState.FATAL_FAILURE) { - callback(new Error('Failed to connect to server')); - } else { - client.$channel.watchConnectivityState(new_state, deadline, checkState); - } - }; - checkState(); + Client.prototype.waitForReady.apply(client, deadline, callback); }; /** diff --git a/src/node/src/common.js b/src/node/src/common.js index 757969dbddb..4dad60e630f 100644 --- a/src/node/src/common.js +++ b/src/node/src/common.js @@ -43,7 +43,8 @@ var _ = require('lodash'); /** * Wrap a function to pass null-like values through without calling it. If no - * function is given, just uses the identity; + * function is given, just uses the identity. + * @private * @param {?function} func The function to wrap * @return {function} The wrapped function */ @@ -90,3 +91,67 @@ exports.defaultGrpcOptions = { enumsAsStrings: true, deprecatedArgumentOrder: false }; + +// JSDoc definitions that are used in multiple other modules + +/** + * The EventEmitter class in the event standard module + * @external EventEmitter + * @see https://nodejs.org/api/events.html#events_class_eventemitter + */ + +/** + * The Readable class in the stream standard module + * @external Readable + * @see https://nodejs.org/api/stream.html#stream_readable_streams + */ + +/** + * The Writable class in the stream standard module + * @external Writable + * @see https://nodejs.org/api/stream.html#stream_writable_streams + */ + +/** + * The Duplex class in the stream standard module + * @external Duplex + * @see https://nodejs.org/api/stream.html#stream_class_stream_duplex + */ + +/** + * A serialization function + * @callback module:src/common~serialize + * @param {*} value The value to serialize + * @return {Buffer} The value serialized as a byte sequence + */ + +/** + * A deserialization function + * @callback module:src/common~deserialize + * @param {Buffer} data The byte sequence to deserialize + * @return {*} The data deserialized as a value + */ + +/** + * An object that completely defines a service method signature. + * @typedef {Object} module:src/common~MethodDefinition + * @property {string} path The method's URL path + * @property {boolean} requestStream Indicates whether the method accepts + * a stream of requests + * @property {boolean} responseStream Indicates whether the method returns + * a stream of responses + * @property {module:src/common~serialize} requestSerialize Serialization + * function for request values + * @property {module:src/common~serialize} responseSerialize Serialization + * function for response values + * @property {module:src/common~deserialize} requestDeserialize Deserialization + * function for request data + * @property {module:src/common~deserialize} responseDeserialize Deserialization + * function for repsonse data + */ + +/** + * An object that completely defines a service. + * @typedef {Object.} + * module:src/common~ServiceDefinition + */ diff --git a/src/node/src/metadata.js b/src/node/src/metadata.js index 612361b0eaf..92cf23998b3 100644 --- a/src/node/src/metadata.js +++ b/src/node/src/metadata.js @@ -35,12 +35,7 @@ * Metadata module * * This module defines the Metadata class, which represents header and trailer - * metadata for gRPC calls. Here is an example of how to use it: - * - * var metadata = new metadata_module.Metadata(); - * metadata.set('key1', 'value1'); - * metadata.add('key1', 'value2'); - * metadata.get('key1') // returns ['value1', 'value2'] + * metadata for gRPC calls. * * @module */ @@ -54,6 +49,12 @@ var grpc = require('./grpc_extension'); /** * Class for storing metadata. Keys are normalized to lowercase ASCII. * @constructor + * @alias module:src/metadata.Metadata + * @example + * var metadata = new metadata_module.Metadata(); + * metadata.set('key1', 'value1'); + * metadata.add('key1', 'value2'); + * metadata.get('key1') // returns ['value1', 'value2'] */ function Metadata() { this._internal_repr = {}; From 1d00ccc3e938f8e19506c2ef483c6521c6f504e7 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 May 2017 16:28:38 +0200 Subject: [PATCH 141/177] fix C# macos x86 artifact build --- tools/run_tests/artifacts/artifact_targets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 04702baccae..c90da066c34 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -202,6 +202,7 @@ class CSharpExtArtifact: 'EMBED_OPENSSL': 'true', 'EMBED_ZLIB': 'true', 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', + 'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', 'LDFLAGS': ''} if self.platform == 'linux': return create_docker_jobspec(self.name, @@ -211,6 +212,7 @@ class CSharpExtArtifact: else: archflag = _ARCH_FLAG_MAP[self.arch] environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) + environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) environ['LDFLAGS'] += ' %s' % archflag return create_jobspec(self.name, ['tools/run_tests/artifacts/build_artifact_csharp.sh'], From 93f307a9ba6ca01ba43ef7bdbff39e687450d40d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:26:22 -0700 Subject: [PATCH 142/177] Fix trace dependency mess --- src/core/lib/debug/trace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/debug/trace.c b/src/core/lib/debug/trace.c index 54653868b83..4dfea44c57e 100644 --- a/src/core/lib/debug/trace.c +++ b/src/core/lib/debug/trace.c @@ -35,11 +35,12 @@ #include -#include #include #include #include "src/core/lib/support/env.h" +int grpc_tracer_set_enabled(const char *name, int enabled); + typedef struct tracer { const char *name; grpc_tracer_flag *flag; From f70fe1dfc4779970fb43a70814a8bfd8307e264e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:27:23 -0700 Subject: [PATCH 143/177] Fix mac build --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index ab23c2feaa8..d31d4631a0a 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -1328,7 +1328,7 @@ const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( #include "src/core/lib/iomgr/ev_posix.h" /* If GRPC_LINUX_EPOLL is not defined, it means epoll is not available. Return * NULL */ -const grpc_event_engine_vtable *grpc_init_epoll_linux( +const grpc_event_engine_vtable *grpc_init_epoll_thread_pool_linux( bool requested_explicitly) { return NULL; } From 020176dd5c9cbec8b32348271e39fa6121b7b2f9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:36:44 -0700 Subject: [PATCH 144/177] Fix tracer decl --- src/core/lib/iomgr/tcp_uv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c index 8e8db9f7b45..aba8136ccb1 100644 --- a/src/core/lib/iomgr/tcp_uv.c +++ b/src/core/lib/iomgr/tcp_uv.c @@ -52,7 +52,7 @@ #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" -int grpc_tcp_trace = 0; +grpc_tracer_flag grpc_tcp_trace = GRPC_TRACER_INITIALIZER(false); typedef struct { grpc_endpoint base; From 341bcc581294b703e9413bcad200541bb6833989 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:37:44 -0700 Subject: [PATCH 145/177] Fix tracer usage --- src/core/lib/iomgr/tcp_uv.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/lib/iomgr/tcp_uv.c b/src/core/lib/iomgr/tcp_uv.c index aba8136ccb1..e7157537f6d 100644 --- a/src/core/lib/iomgr/tcp_uv.c +++ b/src/core/lib/iomgr/tcp_uv.c @@ -158,7 +158,7 @@ static void read_callback(uv_stream_t *stream, ssize_t nread, sub = grpc_slice_sub_no_ref(tcp->read_slice, 0, (size_t)nread); grpc_slice_buffer_add(tcp->read_slices, sub); error = GRPC_ERROR_NONE; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t i; const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "read: error=%s", str); @@ -199,7 +199,7 @@ static void uv_endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice_from_static_string(uv_strerror(status))); grpc_closure_sched(exec_ctx, cb, error); } - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "Initiating read on %p: error=%s", tcp, str); } @@ -217,7 +217,7 @@ static void write_callback(uv_write_t *req, int status) { } else { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("TCP Write failed"); } - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { const char *str = grpc_error_string(error); gpr_log(GPR_DEBUG, "write complete on %p: error=%s", tcp, str); } @@ -238,7 +238,7 @@ static void uv_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice *slice; uv_write_t *write_req; - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { size_t j; for (j = 0; j < write_slices->count; j++) { @@ -346,7 +346,7 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle, char *peer_string) { grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); - if (grpc_tcp_trace) { + if (GRPC_TRACER_ON(grpc_tcp_trace)) { gpr_log(GPR_DEBUG, "Creating TCP endpoint %p", tcp); } From db3462790762ec201ed384ba645cd2e36fcafa56 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:37:52 -0700 Subject: [PATCH 146/177] Fix test name --- test/core/iomgr/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index 0cf93e73f5c..808faf57c9d 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -55,8 +55,8 @@ cc_test( ) cc_test( - name = "ev_epoll_linux_test", - srcs = ["ev_epoll_linux_test.c"], + name = "ev_epollsig_linux_test", + srcs = ["ev_epollsig_linux_test.c"], deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], copts = ['-std=c99'] ) From 9cd102a4ebf7d8a76ba481748b20e450929bd65b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 08:38:41 -0700 Subject: [PATCH 147/177] Fix printf --- src/core/lib/iomgr/ev_epoll_thread_pool_linux.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c index d31d4631a0a..bb44321922a 100644 --- a/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c +++ b/src/core/lib/iomgr/ev_epoll_thread_pool_linux.c @@ -1167,7 +1167,8 @@ static void add_fd_to_eps(grpc_fd *fd) { EPS_ADD_REF(eps, "fd"); fd->eps = eps; - GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %ld)", fd->fd, idx); + GRPC_POLLING_TRACE("add_fd_to_eps (fd: %d, eps idx = %" PRIdPTR ")", fd->fd, + idx); gpr_mu_unlock(&fd->mu); GRPC_LOG_IF_ERROR("add_fd_to_eps", error); From 0725d3a4b385f0213a1058037443f5c016b823c4 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 9 May 2017 12:05:11 -0700 Subject: [PATCH 148/177] Run bm resonable number of times --- .../microbenchmarks/bm_fullstack_trickle.cc | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index b0f7b34bb50..db07a553e3b 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -404,15 +404,18 @@ static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { } static void UnaryTrickleArgs(benchmark::internal::Benchmark* b) { - for (int i = 1; i <= 128 * 1024 * 1024; i *= 32) { - for (int j = 1; j <= 128 * 1024 * 1024; j *= 32) { - for (int k = 64; k <= 128 * 1024 * 1024; k *= 16) { - double expected_time = - static_cast(14 + i + k) / (125.0 * 2 * static_cast(j)); - if (expected_time > 2.0) continue; - b->Args({i, j, k}); - } - } + const int cli_1024k = 1024 * 1024; + const int cli_32M = 32 * 1024 * 1024; + const int svr_256k = 256 * 1024; + const int svr_4M = 4 * 1024 * 1024; + const int svr_64M = 64 * 1024 * 1024; + for (int bw = 64; bw <= 128 * 1024 * 1024; bw *= 16) { + b->Args({bw, cli_1024k, svr_256k}); + b->Args({bw, cli_1024k, svr_4M}); + b->Args({bw, cli_1024k, svr_64M}); + b->Args({bw, cli_32M, svr_256k}); + b->Args({bw, cli_32M, svr_4M}); + b->Args({bw, cli_32M, svr_64M}); } } BENCHMARK(BM_PumpUnbalancedUnary_Trickle)->Apply(UnaryTrickleArgs); From cc00300c93d526391fe7a1070cc64af636bb1a23 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 12:30:11 -0700 Subject: [PATCH 149/177] Add missing file to BUILD --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index b19ecec62f4..aae7b6ffbec 100644 --- a/BUILD +++ b/BUILD @@ -605,6 +605,7 @@ grpc_cc_library( "src/core/lib/iomgr/error.h", "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.h", + "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", "src/core/lib/iomgr/is_epollexclusive_available.h", "src/core/lib/iomgr/sys_epoll_wrapper.h", From 99ee8d53085d10a2d008f394321cf6fdb9623d8b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 12:36:20 -0700 Subject: [PATCH 150/177] Solve uv link problem --- src/core/lib/iomgr/timer_uv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 8e8a07578c6..3830d78ba86 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -42,6 +42,9 @@ #include +grpc_tracer_flag grpc_timer_trace = GRPC_TRACER_INITIALIZER(false); +grpc_tracer_flag grpc_timer_check_trace = GRPC_TRACER_INITIALIZER(false); + static void timer_close_callback(uv_handle_t *handle) { gpr_free(handle); } static void stop_uv_timer(uv_timer_t *handle) { From b91c5fb8ba84b193a0b2e586e4dca05978ffad32 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 9 May 2017 13:28:32 -0700 Subject: [PATCH 151/177] Fix python artifact build --- setup.py | 2 +- src/python/grpcio/commands.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 18ba802fb0f..adf8387d3db 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ if EXTRA_ENV_COMPILE_ARGS is None: elif 'win32' in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC' elif "linux" in sys.platform: - EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -fvisibility=hidden -fno-wrapv' + EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -std=gnu99 -fvisibility=hidden -fno-wrapv' elif "darwin" in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -fvisibility=hidden -fno-wrapv' diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 4f072809c47..f4ccb1ab94c 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -284,9 +284,9 @@ class BuildExt(build_ext.build_ext): stderr=subprocess.PIPE) make_out, make_err = make_process.communicate() if make_out and make_process.returncode != 0: - sys.stdout.write(make_out + '\n') + sys.stdout.write(str(make_out) + '\n') if make_err: - sys.stderr.write(make_err + '\n') + sys.stderr.write(str(make_err) + '\n') if make_process.returncode != 0: raise Exception("make command failed!") From f2f125a38ac23a42fe7c60d174c380d1d7e4c7d0 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 9 May 2017 15:10:51 -0700 Subject: [PATCH 152/177] Switch 'apply' for 'call' in pass-through functions --- src/node/src/client.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/node/src/client.js b/src/node/src/client.js index 75567c69ecd..43502da6af6 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -899,7 +899,7 @@ exports.makeClientConstructor = function(methods, serviceName, * @return {Channel} The channel */ exports.getClientChannel = function(client) { - return Client.prototype.getChannel.apply(client); + return Client.prototype.getChannel.call(client); }; /** @@ -915,7 +915,7 @@ exports.getClientChannel = function(client) { * to connect. */ exports.waitForClientReady = function(client, deadline, callback) { - Client.prototype.waitForReady.apply(client, deadline, callback); + Client.prototype.waitForReady.call(client, deadline, callback); }; /** From 8ce9a468042c45c84476d53f287b95421cc111e5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 9 May 2017 15:31:54 -0700 Subject: [PATCH 153/177] Fix small message streaming ping-pong write count regression --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- src/core/ext/transport/chttp2/transport/parsing.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 30738080ee7..3c5216e1040 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -2554,7 +2554,7 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, add_max_recv_bytes); if ((int64_t)s->incoming_window_delta + (int64_t)initial_window_size - (int64_t)s->announce_window > - 2 * (int64_t)initial_window_size) { + (int64_t)initial_window_size / 2) { write_type = GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK; } grpc_chttp2_become_writable(exec_ctx, t, s, write_type, diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index d0b94bd6ced..b0dd6851162 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -418,7 +418,10 @@ static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA("parse", t, s, incoming_frame_size); - if ((int64_t)s->incoming_window_delta - (int64_t)s->announce_window <= 0) { + if ((int64_t)s->incoming_window_delta - (int64_t)s->announce_window <= + -(int64_t)t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] / + 2) { grpc_chttp2_become_writable(exec_ctx, t, s, GRPC_CHTTP2_STREAM_WRITE_INITIATE_UNCOVERED, "window-update-required"); From c7de5a760109cdb048552aeabc9b68b238f1843f Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 9 May 2017 15:50:32 -0700 Subject: [PATCH 154/177] Add warmup --- .../microbenchmarks/bm_fullstack_trickle.cc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index db07a553e3b..05fea06f44d 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -53,7 +53,7 @@ DEFINE_int32( "Number of megabytes to pump before collecting flow control stats"); DEFINE_int32( warmup_iterations, 100, - "Number of megabytes to pump before collecting flow control stats"); + "Number of iterations to run before collecting flow control stats"); DEFINE_int32(warmup_max_time_seconds, 10, "Maximum number of seconds to run warmup loop"); @@ -366,7 +366,7 @@ static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { fixture->cq(), tag(1)); std::unique_ptr stub( EchoTestService::NewStub(fixture->channel())); - while (state.KeepRunning()) { + auto inner_loop = [&](bool in_warmup) { GPR_TIMER_SCOPE("BenchmarkCycle", 0); recv_response.Clear(); ClientContext cli_ctx; @@ -394,6 +394,21 @@ static void BM_PumpUnbalancedUnary_Trickle(benchmark::State& state) { senv = new (senv) ServerEnv(); service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer, fixture->cq(), fixture->cq(), tag(slot)); + }; + gpr_timespec warmup_start = gpr_now(GPR_CLOCK_MONOTONIC); + for (int i = 0; + i < GPR_MAX(FLAGS_warmup_iterations, FLAGS_warmup_megabytes * 1024 * + 1024 / (14 + state.range(0))); + i++) { + inner_loop(true); + if (gpr_time_cmp(gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), warmup_start), + gpr_time_from_seconds(FLAGS_warmup_max_time_seconds, + GPR_TIMESPAN)) > 0) { + break; + } + } + while (state.KeepRunning()) { + inner_loop(false); } fixture->Finish(state); fixture.reset(); From d586587d85d87e996b0131638d338dd9b484985b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 07:26:20 -0700 Subject: [PATCH 155/177] Fix BUILD --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index aae7b6ffbec..0a188a82d37 100644 --- a/BUILD +++ b/BUILD @@ -479,6 +479,7 @@ grpc_cc_library( "src/core/lib/iomgr/endpoint_pair_windows.c", "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/ev_epoll1_linux.c", + "src/core/lib/iomgr/ev_epollsig_linux.c", "src/core/lib/iomgr/ev_epollex_linux.c", "src/core/lib/iomgr/is_epollexclusive_available.c", "src/core/lib/iomgr/ev_epoll_thread_pool_linux.c", From cdc97a1226cc332ac3c71fd09aa04b6295ec9bb2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 07:32:12 -0700 Subject: [PATCH 156/177] Fix tests on Mac --- test/core/iomgr/ev_epollsig_linux_test.c | 1 + test/core/iomgr/fd_conservation_posix_test.c | 1 + test/core/iomgr/fd_posix_test.c | 1 + test/core/iomgr/pollset_set_test.c | 1 + test/core/iomgr/resolve_address_posix_test.c | 1 + test/core/iomgr/resolve_address_test.c | 1 + 6 files changed, 6 insertions(+) diff --git a/test/core/iomgr/ev_epollsig_linux_test.c b/test/core/iomgr/ev_epollsig_linux_test.c index efc8ed36be6..45c542de4e4 100644 --- a/test/core/iomgr/ev_epollsig_linux_test.c +++ b/test/core/iomgr/ev_epollsig_linux_test.c @@ -403,6 +403,7 @@ int main(int argc, char **argv) { const char *poll_strategy = NULL; grpc_test_init(argc, argv); grpc_iomgr_init(); + grpc_iomgr_start(); poll_strategy = grpc_get_poll_strategy_name(); if (poll_strategy != NULL && strcmp(poll_strategy, "epollsig") == 0) { diff --git a/test/core/iomgr/fd_conservation_posix_test.c b/test/core/iomgr/fd_conservation_posix_test.c index 6ac322bb014..f6620706559 100644 --- a/test/core/iomgr/fd_conservation_posix_test.c +++ b/test/core/iomgr/fd_conservation_posix_test.c @@ -46,6 +46,7 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_iomgr_init(); + grpc_iomgr_start(); /* set max # of file descriptors to a low value, and verify we can create and destroy many more than this number diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c index d6cee8e43a7..9e8fe8bffad 100644 --- a/test/core/iomgr/fd_posix_test.c +++ b/test/core/iomgr/fd_posix_test.c @@ -543,6 +543,7 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); + grpc_iomgr_start(); g_pollset = gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(g_pollset, &g_mu); test_grpc_fd(); diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 9755a7e5526..092711381d1 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -448,6 +448,7 @@ int main(int argc, char **argv) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); grpc_iomgr_init(); + grpc_iomgr_start(); if (poll_strategy != NULL && (strcmp(poll_strategy, "epoll") == 0 || diff --git a/test/core/iomgr/resolve_address_posix_test.c b/test/core/iomgr/resolve_address_posix_test.c index 7fba0b92bec..bee7036ec83 100644 --- a/test/core/iomgr/resolve_address_posix_test.c +++ b/test/core/iomgr/resolve_address_posix_test.c @@ -176,6 +176,7 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_executor_init(); grpc_iomgr_init(); + grpc_iomgr_start(); test_unix_socket(); test_unix_socket_path_name_too_long(); { diff --git a/test/core/iomgr/resolve_address_test.c b/test/core/iomgr/resolve_address_test.c index 0425c3b3f5a..83f73070dcc 100644 --- a/test/core/iomgr/resolve_address_test.c +++ b/test/core/iomgr/resolve_address_test.c @@ -265,6 +265,7 @@ int main(int argc, char **argv) { grpc_test_init(argc, argv); grpc_executor_init(); grpc_iomgr_init(); + grpc_iomgr_start(); test_localhost(); test_default_port(); test_non_numeric_default_port(); From 74bf087e9c28a6cb6ee654990eae71df87332aba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 07:44:04 -0700 Subject: [PATCH 157/177] Increase error threshold for too many writes --- test/cpp/performance/writes_per_rpc_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/performance/writes_per_rpc_test.cc b/test/cpp/performance/writes_per_rpc_test.cc index 7a914c15474..12d8268330c 100644 --- a/test/cpp/performance/writes_per_rpc_test.cc +++ b/test/cpp/performance/writes_per_rpc_test.cc @@ -254,8 +254,8 @@ TEST(WritesPerRpcTest, UnaryPingPong) { EXPECT_LT(UnaryPingPong(0, 0), 2.05); EXPECT_LT(UnaryPingPong(1, 0), 2.05); EXPECT_LT(UnaryPingPong(0, 1), 2.05); - EXPECT_LT(UnaryPingPong(4096, 0), 2.2); - EXPECT_LT(UnaryPingPong(0, 4096), 2.2); + EXPECT_LT(UnaryPingPong(4096, 0), 2.5); + EXPECT_LT(UnaryPingPong(0, 4096), 2.5); } } // namespace testing From 1b84e038848277fbda554cabc75a3ea05f97bcc7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 07:56:42 -0700 Subject: [PATCH 158/177] Dont run ubsan tests on boringssl --- src/boringssl/gen_build_yaml.py | 2 +- tools/run_tests/generated/tests.json | 195 ++++++++++++++++++--------- 2 files changed, 131 insertions(+), 66 deletions(-) diff --git a/src/boringssl/gen_build_yaml.py b/src/boringssl/gen_build_yaml.py index c53beb0da5f..d01c2b4ec55 100755 --- a/src/boringssl/gen_build_yaml.py +++ b/src/boringssl/gen_build_yaml.py @@ -138,7 +138,7 @@ class Grpc(object): { 'name': 'boringssl_%s' % os.path.basename(test[0]), 'args': [map_testarg(arg) for arg in test[1:]], - 'exclude_configs': ['asan'], + 'exclude_configs': ['asan', 'ubsan'], 'ci_platforms': ['linux', 'mac', 'posix', 'windows'], 'platforms': ['linux', 'mac', 'posix', 'windows'], 'flaky': False, diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a004e8d945d..c77961c0124 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4205,7 +4205,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4229,7 +4230,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4253,7 +4255,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4277,7 +4280,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4303,7 +4307,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4327,7 +4332,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4351,7 +4357,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4378,7 +4385,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4405,7 +4413,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4432,7 +4441,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4459,7 +4469,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4486,7 +4497,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4513,7 +4525,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4540,7 +4553,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4567,7 +4581,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4594,7 +4609,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4621,7 +4637,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4648,7 +4665,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4675,7 +4693,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4702,7 +4721,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4729,7 +4749,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4756,7 +4777,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4783,7 +4805,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4810,7 +4833,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4837,7 +4861,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4864,7 +4889,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4891,7 +4917,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4917,7 +4944,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4941,7 +4969,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4965,7 +4994,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -4991,7 +5021,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5015,7 +5046,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5039,7 +5071,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5063,7 +5096,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5087,7 +5121,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5111,7 +5146,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5135,7 +5171,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5159,7 +5196,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5185,7 +5223,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5211,7 +5250,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5235,7 +5275,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5261,7 +5302,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5285,7 +5327,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5309,7 +5352,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5335,7 +5379,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5359,7 +5404,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5383,7 +5429,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5409,7 +5456,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5433,7 +5481,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5457,7 +5506,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5481,7 +5531,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5505,7 +5556,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5531,7 +5583,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5555,7 +5608,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5579,7 +5633,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5603,7 +5658,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5629,7 +5685,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5653,7 +5710,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5677,7 +5735,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5701,7 +5760,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5725,7 +5785,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5749,7 +5810,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5773,7 +5835,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5797,7 +5860,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", @@ -5821,7 +5885,8 @@ "cpu_cost": 1.0, "defaults": "boringssl", "exclude_configs": [ - "asan" + "asan", + "ubsan" ], "flaky": false, "language": "c++", From 7b622ab558882a984b597cd87fe7175f5e1eea23 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 07:58:41 -0700 Subject: [PATCH 159/177] ubsan fix: dont deref past end of array --- test/cpp/microbenchmarks/bm_call_create.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index c91219e98c8..67e7c025351 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -563,7 +563,8 @@ static void BM_IsolatedFilter(benchmark::State &state) { } grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - size_t channel_size = grpc_channel_stack_size(&filters[0], filters.size()); + size_t channel_size = grpc_channel_stack_size( + filters.size() == 0 ? NULL : &filters[0], filters.size()); grpc_channel_stack *channel_stack = static_cast(gpr_zalloc(channel_size)); GPR_ASSERT(GRPC_LOG_IF_ERROR( From a189bacc9517919546b97e02aba5c01224f6790f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 08:04:38 -0700 Subject: [PATCH 160/177] Add protobuf failures to ubsan suppression file --- tools/ubsan_suppressions.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ubsan_suppressions.txt b/tools/ubsan_suppressions.txt index 9869f98a22e..f87ed185654 100644 --- a/tools/ubsan_suppressions.txt +++ b/tools/ubsan_suppressions.txt @@ -4,4 +4,6 @@ nonnull-attribute:CBB_add_bytes nonnull-attribute:rsa_blinding_get nonnull-attribute:ssl_copy_key_material alignment:CRYPTO_cbc128_encrypt +nonnull-attribute:google::protobuf::DescriptorBuilder::BuildFileImpl +nonnull-attribute:google::protobuf::TextFormat::Printer::TextGenerator::Write From 4d16f4dd1db5e0546a1f171496f1f3dba888eab1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 08:21:08 -0700 Subject: [PATCH 161/177] Add some casts for C++ compatibility --- test/core/end2end/cq_verifier.c | 4 +-- test/core/end2end/fake_resolver.c | 15 ++++++----- .../end2end/fixtures/http_proxy_fixture.c | 27 ++++++++++--------- test/core/security/oauth2_utils.c | 6 ++--- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/test/core/end2end/cq_verifier.c b/test/core/end2end/cq_verifier.c index 5eea5d43fe0..0fafb0c8c99 100644 --- a/test/core/end2end/cq_verifier.c +++ b/test/core/end2end/cq_verifier.c @@ -77,7 +77,7 @@ struct cq_verifier { }; cq_verifier *cq_verifier_create(grpc_completion_queue *cq) { - cq_verifier *v = gpr_malloc(sizeof(cq_verifier)); + cq_verifier *v = (cq_verifier *)gpr_malloc(sizeof(cq_verifier)); v->cq = cq; v->first_expectation = NULL; return v; @@ -314,7 +314,7 @@ void cq_verify_empty(cq_verifier *v) { cq_verify_empty_timeout(v, 1); } static void add(cq_verifier *v, const char *file, int line, grpc_completion_type type, void *tag, bool success) { - expectation *e = gpr_malloc(sizeof(expectation)); + expectation *e = (expectation *)gpr_malloc(sizeof(expectation)); e->type = type; e->file = file; e->line = line; diff --git a/test/core/end2end/fake_resolver.c b/test/core/end2end/fake_resolver.c index df902a24bff..736b224fd60 100644 --- a/test/core/end2end/fake_resolver.c +++ b/test/core/end2end/fake_resolver.c @@ -136,7 +136,7 @@ struct grpc_fake_resolver_response_generator { grpc_fake_resolver_response_generator* grpc_fake_resolver_response_generator_create() { grpc_fake_resolver_response_generator* generator = - gpr_zalloc(sizeof(*generator)); + (grpc_fake_resolver_response_generator*)gpr_zalloc(sizeof(*generator)); gpr_ref_init(&generator->refcount, 1); return generator; } @@ -157,7 +157,8 @@ void grpc_fake_resolver_response_generator_unref( static void set_response_cb(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - grpc_fake_resolver_response_generator* generator = arg; + grpc_fake_resolver_response_generator* generator = + (grpc_fake_resolver_response_generator*)arg; fake_resolver* r = generator->resolver; if (r->next_results != NULL) { grpc_channel_args_destroy(exec_ctx, r->next_results); @@ -180,11 +181,13 @@ void grpc_fake_resolver_response_generator_set_response( } static void* response_generator_arg_copy(void* p) { - return grpc_fake_resolver_response_generator_ref(p); + return grpc_fake_resolver_response_generator_ref( + (grpc_fake_resolver_response_generator*)p); } static void response_generator_arg_destroy(grpc_exec_ctx* exec_ctx, void* p) { - grpc_fake_resolver_response_generator_unref(p); + grpc_fake_resolver_response_generator_unref( + (grpc_fake_resolver_response_generator*)p); } static int response_generator_cmp(void* a, void* b) { return GPR_ICMP(a, b); } @@ -208,7 +211,7 @@ grpc_fake_resolver_get_response_generator(const grpc_channel_args* args) { const grpc_arg* arg = grpc_channel_args_find(args, GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR); if (arg == NULL || arg->type != GRPC_ARG_POINTER) return NULL; - return arg->value.pointer.p; + return (grpc_fake_resolver_response_generator*)arg->value.pointer.p; } // @@ -222,7 +225,7 @@ static void fake_resolver_factory_unref(grpc_resolver_factory* factory) {} static grpc_resolver* fake_resolver_create(grpc_exec_ctx* exec_ctx, grpc_resolver_factory* factory, grpc_resolver_args* args) { - fake_resolver* r = gpr_zalloc(sizeof(*r)); + fake_resolver* r = (fake_resolver*)gpr_zalloc(sizeof(*r)); r->channel_args = grpc_channel_args_copy(args->args); grpc_resolver_init(&r->base, &fake_resolver_vtable, args->combiner); grpc_fake_resolver_response_generator* response_generator = diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index f0d09487c62..5df43b93847 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -156,7 +156,7 @@ static void proxy_connection_failed(grpc_exec_ctx* exec_ctx, // Callback for writing proxy data to the client. static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy client write", error); @@ -181,7 +181,7 @@ static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg, // Callback for writing proxy data to the backend server. static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, false /* is_client */, "HTTP proxy server write", error); @@ -207,7 +207,7 @@ static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg, // the backend server. static void on_client_read_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy client read", error); @@ -239,7 +239,7 @@ static void on_client_read_done(grpc_exec_ctx* exec_ctx, void* arg, // proxied to the client. static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, false /* is_client */, "HTTP proxy server read", error); @@ -270,7 +270,7 @@ static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg, // Callback to write the HTTP response for the CONNECT request. static void on_write_response_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy write response", error); @@ -294,7 +294,7 @@ static void on_write_response_done(grpc_exec_ctx* exec_ctx, void* arg, // CONNECT request. static void on_server_connect_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; if (error != GRPC_ERROR_NONE) { // TODO(roth): Technically, in this case, we should handle the error // by returning an HTTP response to the client indicating that the @@ -324,7 +324,7 @@ static void on_server_connect_done(grpc_exec_ctx* exec_ctx, void* arg, // which will cause the client connection to be dropped. static void on_read_request_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - proxy_connection* conn = arg; + proxy_connection* conn = (proxy_connection*)arg; gpr_log(GPR_DEBUG, "on_read_request_done: %p %s", conn, grpc_error_string(error)); if (error != GRPC_ERROR_NONE) { @@ -389,9 +389,9 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg, grpc_endpoint* endpoint, grpc_pollset* accepting_pollset, grpc_tcp_server_acceptor* acceptor) { gpr_free(acceptor); - grpc_end2end_http_proxy* proxy = arg; + grpc_end2end_http_proxy* proxy = (grpc_end2end_http_proxy*)arg; // Instantiate proxy_connection. - proxy_connection* conn = gpr_zalloc(sizeof(*conn)); + proxy_connection* conn = (proxy_connection*)gpr_zalloc(sizeof(*conn)); gpr_ref(&proxy->users); conn->client_endpoint = endpoint; conn->proxy = proxy; @@ -430,7 +430,7 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg, // static void thread_main(void* arg) { - grpc_end2end_http_proxy* proxy = arg; + grpc_end2end_http_proxy* proxy = (grpc_end2end_http_proxy*)arg; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; do { gpr_ref(&proxy->users); @@ -450,7 +450,8 @@ static void thread_main(void* arg) { grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_end2end_http_proxy* proxy = gpr_malloc(sizeof(*proxy)); + grpc_end2end_http_proxy* proxy = + (grpc_end2end_http_proxy*)gpr_malloc(sizeof(*proxy)); memset(proxy, 0, sizeof(*proxy)); gpr_ref_init(&proxy->users, 1); // Construct proxy address. @@ -473,7 +474,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { GPR_ASSERT(error == GRPC_ERROR_NONE); GPR_ASSERT(port == proxy_port); // Start server. - proxy->pollset = gpr_zalloc(grpc_pollset_size()); + proxy->pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(proxy->pollset, &proxy->mu); grpc_tcp_server_start(&exec_ctx, proxy->server, &proxy->pollset, 1, on_accept, proxy); @@ -487,7 +488,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { static void destroy_pollset(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { - grpc_pollset* pollset = arg; + grpc_pollset* pollset = (grpc_pollset*)arg; grpc_pollset_destroy(pollset); gpr_free(pollset); } diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c index f0550db1d06..838625705dd 100644 --- a/test/core/security/oauth2_utils.c +++ b/test/core/security/oauth2_utils.c @@ -55,7 +55,7 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, grpc_credentials_md *md_elems, size_t num_md, grpc_credentials_status status, const char *error_details) { - oauth2_request *request = user_data; + oauth2_request *request = (oauth2_request *)user_data; char *token = NULL; grpc_slice token_slice; if (status == GRPC_CREDENTIALS_ERROR) { @@ -63,7 +63,7 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data, } else { GPR_ASSERT(num_md == 1); token_slice = md_elems[0].value; - token = gpr_malloc(GRPC_SLICE_LENGTH(token_slice) + 1); + token = (char *)gpr_malloc(GRPC_SLICE_LENGTH(token_slice) + 1); memcpy(token, GRPC_SLICE_START_PTR(token_slice), GRPC_SLICE_LENGTH(token_slice)); token[GRPC_SLICE_LENGTH(token_slice)] = '\0'; @@ -87,7 +87,7 @@ char *grpc_test_fetch_oauth2_token_with_credentials( grpc_closure do_nothing_closure; grpc_auth_metadata_context null_ctx = {"", "", NULL, NULL}; - grpc_pollset *pollset = gpr_zalloc(grpc_pollset_size()); + grpc_pollset *pollset = (grpc_pollset *)gpr_zalloc(grpc_pollset_size()); grpc_pollset_init(pollset, &request.mu); request.pops = grpc_polling_entity_create_from_pollset(pollset); request.is_done = 0; From 15285fbbf90964fb4e04727478207384390497ef Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 08:29:42 -0700 Subject: [PATCH 162/177] Add missing include --- src/core/lib/iomgr/timer_uv.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 3830d78ba86..8a4c01c4c5d 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -38,6 +38,7 @@ #include #include +#include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/timer.h" #include From 1842c4379cc863a61269e083045522eb6542036d Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 10 May 2017 08:54:39 -0700 Subject: [PATCH 163/177] Clang --- test/cpp/microbenchmarks/bm_fullstack_trickle.cc | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc index 05fea06f44d..6b9fa8b38da 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc @@ -77,17 +77,14 @@ static void write_csv(std::ostream* out, A0&& a0, Arg&&... arg) { class TrickledCHTTP2 : public EndpointPairFixture { public: - TrickledCHTTP2(Service* service, bool streaming, size_t req_size, size_t resp_size, - size_t kilobits_per_second) + TrickledCHTTP2(Service* service, bool streaming, size_t req_size, + size_t resp_size, size_t kilobits_per_second) : EndpointPairFixture(service, MakeEndpoints(kilobits_per_second), FixtureConfiguration()) { if (FLAGS_log) { std::ostringstream fn; - fn << "trickle." << (streaming ? "streaming" - : "unary" ) - << "." << req_size << "." << resp_size - << "." << kilobits_per_second - << ".csv"; + fn << "trickle." << (streaming ? "streaming" : "unary") << "." << req_size + << "." << resp_size << "." << kilobits_per_second << ".csv"; log_.reset(new std::ofstream(fn.str().c_str())); write_csv(log_.get(), "t", "iteration", "client_backlog", "server_backlog", "client_t_stall", "client_s_stall", From 22dbd73b4368cfce188b6f85505b287e710b4f11 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 10 May 2017 09:16:22 -0700 Subject: [PATCH 164/177] Add static assert to enforce subtype invariant --- include/grpc++/impl/codegen/proto_utils.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/grpc++/impl/codegen/proto_utils.h b/include/grpc++/impl/codegen/proto_utils.h index a6e10cc8b66..5cdb2a7a155 100644 --- a/include/grpc++/impl/codegen/proto_utils.h +++ b/include/grpc++/impl/codegen/proto_utils.h @@ -183,10 +183,12 @@ class GrpcBufferReader : public ::grpc::protobuf::io::ZeroCopyInputStream { Status status_; }; -// BufferWriter must be a subclass of io::ZeroCopyOutputStream. template Status GenericSerialize(const grpc::protobuf::Message& msg, grpc_byte_buffer** bp, bool* own_buffer) { + static_assert( + std::is_base_of::value, + "BufferWriter must be a subclass of io::ZeroCopyOutputStream"); *own_buffer = true; int byte_size = msg.ByteSize(); if (byte_size <= internal::kGrpcBufferWriterMaxBufferLength) { @@ -205,10 +207,12 @@ Status GenericSerialize(const grpc::protobuf::Message& msg, } } -// BufferReader must be a subclass of io::ZeroCopyInputStream. template Status GenericDeserialize(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg) { + static_assert( + std::is_base_of::value, + "BufferReader must be a subclass of io::ZeroCopyInputStream"); if (buffer == nullptr) { return Status(StatusCode::INTERNAL, "No payload"); } From add2e4da2df304b1331c108f29de218c8f602fde Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 10 May 2017 10:25:13 -0700 Subject: [PATCH 165/177] Remove gcc-4.4 and 4.6 from portability tests since core now accepts C++11. --- .../test/cxx_wheezy_x64/Dockerfile.template | 57 --------- .../cxx_wheezy_x64/post-git-setup.sh.template | 37 ------ .../dockerfile/test/cxx_wheezy_x64/Dockerfile | 117 ------------------ .../test/cxx_wheezy_x64/post-git-setup.sh | 35 ------ tools/run_tests/run_tests.py | 4 - tools/run_tests/run_tests_matrix.py | 13 +- 6 files changed, 1 insertion(+), 262 deletions(-) delete mode 100644 templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template delete mode 100644 templates/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh.template delete mode 100644 tools/dockerfile/test/cxx_wheezy_x64/Dockerfile delete mode 100755 tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh diff --git a/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template deleted file mode 100644 index 6d74f4c8d16..00000000000 --- a/templates/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile.template +++ /dev/null @@ -1,57 +0,0 @@ -%YAML 1.2 ---- | - # Copyright 2016, Google Inc. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are - # met: - # - # * Redistributions of source code must retain the above copyright - # notice, this list of conditions and the following disclaimer. - # * Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following disclaimer - # in the documentation and/or other materials provided with the - # distribution. - # * Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - FROM debian:wheezy - - <%include file="../../apt_get_basic.include"/> - <%include file="../../gcp_api_libraries.include"/> - <%include file="../../python_deps.include"/> - <%include file="../../cxx_deps.include"/> - - RUN apt-get update && apt-get install -y ${'\\'} - gcc-4.4 ${'\\'} - gcc-4.4-multilib ${'\\'} - g++-4.4 ${'\\'} - g++-4.4-multilib - - # set up backport to allow installation of Git version > 1.7 - RUN echo "deb http://http.debian.net/debian wheezy-backports main" \ - >/etc/apt/sources.list.d/wheezy-backports.list - RUN apt-get update -qq - RUN apt-get -t wheezy-backports install -qq git - - RUN wget ${openssl_fallback.base_uri + openssl_fallback.tarball} - - ENV POST_GIT_STEP tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh - - <%include file="../../run_tests_addons.include"/> - # Define the default command. - CMD ["bash"] diff --git a/templates/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh.template b/templates/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh.template deleted file mode 100644 index b8851017484..00000000000 --- a/templates/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh.template +++ /dev/null @@ -1,37 +0,0 @@ -%YAML 1.2 ---- | - #!/bin/bash - # Copyright 2016, Google Inc. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without - # modification, are permitted provided that the following conditions are - # met: - # - # * Redistributions of source code must retain the above copyright - # notice, this list of conditions and the following disclaimer. - # * Redistributions in binary form must reproduce the above - # copyright notice, this list of conditions and the following disclaimer - # in the documentation and/or other materials provided with the - # distribution. - # * Neither the name of Google Inc. nor the names of its - # contributors may be used to endorse or promote products derived from - # this software without specific prior written permission. - # - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - set -ex - - cd /var/local/git/grpc - cp /${openssl_fallback.tarball} third_party - ./tools/openssl/use_openssl.sh diff --git a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile deleted file mode 100644 index 1848e5e5c89..00000000000 --- a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -FROM debian:wheezy - -# Install Git and basic packages. -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - golang \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - perl \ - strace \ - python-dev \ - python-setuptools \ - python-yaml \ - telnet \ - unzip \ - wget \ - zip && apt-get clean - -#================ -# Build profiling -RUN apt-get update && apt-get install -y time && apt-get clean - -# Google Cloud platform API libraries -RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client - -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip - -# Install Python packages from PyPI -RUN pip install pip --upgrade -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 - -#================= -# C++ dependencies -RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean - - -RUN apt-get update && apt-get install -y \ - gcc-4.4 \ - gcc-4.4-multilib \ - g++-4.4 \ - g++-4.4-multilib - -# set up backport to allow installation of Git version > 1.7 -RUN echo "deb http://http.debian.net/debian wheezy-backports main" >/etc/apt/sources.list.d/wheezy-backports.list -RUN apt-get update -qq -RUN apt-get -t wheezy-backports install -qq git - -RUN wget https://openssl.org/source/old/1.0.2/openssl-1.0.2f.tar.gz - -ENV POST_GIT_STEP tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh - -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - - -RUN mkdir /var/local/jenkins - -# Define the default command. -CMD ["bash"] diff --git a/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh b/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh deleted file mode 100755 index dfde93b1bd2..00000000000 --- a/tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -set -ex - -cd /var/local/git/grpc -cp /openssl-1.0.2f.tar.gz third_party -./tools/openssl/use_openssl.sh diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 22891c5c982..27e282285cb 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -390,10 +390,6 @@ class CLanguage(object): if compiler == 'gcc4.9' or compiler == 'default': return ('jessie', []) - elif compiler == 'gcc4.4': - return ('wheezy', self._gcc_make_options(version_suffix='-4.4')) - elif compiler == 'gcc4.6': - return ('wheezy', self._gcc_make_options(version_suffix='-4.6')) elif compiler == 'gcc4.8': return ('jessie', self._gcc_make_options(version_suffix='-4.8')) elif compiler == 'gcc5.3': diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 884d626469d..84551d93948 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -197,20 +197,9 @@ def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS) inner_jobs=inner_jobs) # portability C and C++ on x64 - for compiler in ['gcc4.4', 'gcc4.6', 'gcc5.3', 'gcc_musl', - 'clang3.5', 'clang3.6', 'clang3.7']: - test_jobs += _generate_jobs(languages=['c'], - configs=['dbg'], - platforms=['linux'], - arch='x64', - compiler=compiler, - labels=['portability'], - extra_args=extra_args, - inner_jobs=inner_jobs) - for compiler in ['gcc4.8', 'gcc5.3', 'gcc_musl', 'clang3.5', 'clang3.6', 'clang3.7']: - test_jobs += _generate_jobs(languages=['c++'], + test_jobs += _generate_jobs(languages=['c', 'c++'], configs=['dbg'], platforms=['linux'], arch='x64', From 8683911544f40bb8b2f98a8b1ddc7132d29968fd Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 May 2017 15:22:13 +0200 Subject: [PATCH 166/177] print insecure channel target in trace --- src/core/ext/transport/chttp2/client/insecure/channel_create.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index 9c8505ddfa9..ad674b8eb47 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -101,7 +101,7 @@ grpc_channel *grpc_insecure_channel_create(const char *target, void *reserved) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GRPC_API_TRACE( - "grpc_insecure_channel_create(target=%p, args=%p, reserved=%p)", 3, + "grpc_insecure_channel_create(target=%s, args=%p, reserved=%p)", 3, (target, args, reserved)); GPR_ASSERT(reserved == NULL); // Add channel arg containing the client channel factory. From a1f3c20878b90f1222564c187c67013171f96095 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 May 2017 20:26:41 +0200 Subject: [PATCH 167/177] fix encoding of chars --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 2 +- src/core/ext/transport/chttp2/transport/internal.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 30738080ee7..0cd7c57d84b 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -997,7 +997,7 @@ void grpc_chttp2_add_incoming_goaway(grpc_exec_ctx *exec_ctx, t->seen_goaway = 1; /* When a client receives a GOAWAY with error code ENHANCE_YOUR_CALM and debug - * data equal to “too_many_pings”, it should log the occurrence at a log level + * data equal to "too_many_pings", it should log the occurrence at a log level * that is enabled by default and double the configured KEEPALIVE_TIME used * for new connections on that channel. */ if (t->is_client && goaway_error == GRPC_HTTP2_ENHANCE_YOUR_CALM && diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 733e9900ef1..1f87206114b 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -820,7 +820,7 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, /** Add a new ping strike to ping_recv_state.ping_strikes. If ping_recv_state.ping_strikes > ping_policy.max_ping_strikes, it sends GOAWAY with error code ENHANCE_YOUR_CALM and additional debug data resembling - “too_many_pings” followed by immediately closing the connection. */ + "too_many_pings" followed by immediately closing the connection. */ void grpc_chttp2_add_ping_strike(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t); From 6e18ef3880da523af4c131d34234aa1c4e4221c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 12:54:59 -0700 Subject: [PATCH 168/177] Fix portability --- src/core/lib/iomgr/timer_uv.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/iomgr/timer_uv.c b/src/core/lib/iomgr/timer_uv.c index 8a4c01c4c5d..2952e44b583 100644 --- a/src/core/lib/iomgr/timer_uv.c +++ b/src/core/lib/iomgr/timer_uv.c @@ -104,4 +104,6 @@ bool grpc_timer_check(grpc_exec_ctx *exec_ctx, gpr_timespec now, void grpc_timer_list_init(gpr_timespec now) {} void grpc_timer_list_shutdown(grpc_exec_ctx *exec_ctx) {} +void grpc_timer_consume_kick(void) {} + #endif /* GRPC_UV */ From 518bdfd18c007dc9272b873615f5d077f800c807 Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 10 May 2017 13:38:47 -0700 Subject: [PATCH 169/177] Properly filter the end2end tests --- test/core/end2end/generate_tests.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index dc0925dc9c0..e14157849fb 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -65,7 +65,7 @@ END2END_FIXTURES = { tracing=True), 'h2_ssl': fixture_options(secure=True), 'h2_ssl_cert': fixture_options(secure=True), - 'h2_ssl_proxy': fixture_options(secure=True), + 'h2_ssl_proxy': fixture_options(includes_proxy=True, secure=True), 'h2_uds': fixture_options(dns_resolver=False, platforms=['linux', 'mac', 'posix']), } @@ -95,7 +95,7 @@ END2END_TESTS = { 'cancel_before_invoke': test_options(), 'cancel_in_a_vacuum': test_options(), 'cancel_with_status': test_options(), - 'compressed_payload': test_options(), + 'compressed_payload': test_options(proxyable=False), 'connectivity': test_options(needs_fullstack=True, proxyable=False), 'default_host': test_options(needs_fullstack=True, needs_dns=True), 'disappearing_server': test_options(needs_fullstack=True), @@ -120,7 +120,7 @@ END2END_TESTS = { 'payload': test_options(), 'load_reporting_hook': test_options(), 'ping_pong_streaming': test_options(), - 'ping': test_options(proxyable=False), + 'ping': test_options(needs_fullstack=True, proxyable=False), 'registered_call': test_options(), 'request_with_flags': test_options(proxyable=False), 'request_with_payload': test_options(), From a1d00facbcfcd3a47d5617fd65f6ff5842c15649 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 15:43:44 -0700 Subject: [PATCH 170/177] Finish pumped streams: should alleviate infinite hang --- .../microbenchmarks/bm_fullstack_streaming_pump.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc index 47705d30318..01ff39121e8 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc @@ -105,6 +105,17 @@ static void BM_PumpStreamClientToServer(benchmark::State& state) { GPR_ASSERT(need_tags & (1 << i)); need_tags &= ~(1 << i); } + response_rw.Finish(Status::OK, tag(0)); + Status final_status; + request_rw->Finish(&final_status, tag(1)); + need_tags = (1 << 0) | (1 << 1); + while (need_tags) { + GPR_ASSERT(fixture->cq()->Next(&t, &ok)); + int i = (int)(intptr_t)t; + GPR_ASSERT(need_tags & (1 << i)); + need_tags &= ~(1 << i); + } + GPR_ASSERT(final_status.ok()); } fixture->Finish(state); fixture.reset(); From 2f8cfbde3e47a3c63ce70fd1abd47c6cd98f7d8b Mon Sep 17 00:00:00 2001 From: MakMukhi Date: Wed, 10 May 2017 16:55:30 -0700 Subject: [PATCH 171/177] Post-review updates --- doc/unit_testing.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/unit_testing.md b/doc/unit_testing.md index 3307d115879..380517fea52 100644 --- a/doc/unit_testing.md +++ b/doc/unit_testing.md @@ -2,7 +2,7 @@ tl;dr: [Example code](https://github.com/grpc/grpc/blob/master/test/cpp/end2end/mock_test.cc). -To unit-test client-side logic via the synchronous API, gRPC provides a mocked Stub that can be programmed upon and easily incorporated in the test code. +To unit-test client-side logic via the synchronous API, gRPC provides a mocked Stub based on googletest(googlemock) that can be programmed upon and easily incorporated in the test code. For instance, consider an EchoService like this: @@ -14,7 +14,7 @@ service EchoTestService { } ``` -The code generate would look something like this: +The code generated would look something like this: ```c++ class EchoTestService final { @@ -75,7 +75,7 @@ grpc_proto_library( ``` -By adding such a flag now a header file `echo_mock.h` containing the mocked stub will also be generated. +By adding such a flag now a header file `echo_mock.grpc.pb.h` containing the mocked stub will also be generated. This header file can then be included in test files along with a gmock dependency. From 012e929a1802cb13b0f2a834b9210984af0a8428 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 10 May 2017 21:41:21 -0700 Subject: [PATCH 172/177] Revert "Documentation on how to write unit tests using auto-generated mocked stubs." --- doc/unit_testing.md | 175 -------------------------------------------- 1 file changed, 175 deletions(-) delete mode 100644 doc/unit_testing.md diff --git a/doc/unit_testing.md b/doc/unit_testing.md deleted file mode 100644 index 380517fea52..00000000000 --- a/doc/unit_testing.md +++ /dev/null @@ -1,175 +0,0 @@ -# How to write unit tests for gRPC C++ client. - -tl;dr: [Example code](https://github.com/grpc/grpc/blob/master/test/cpp/end2end/mock_test.cc). - -To unit-test client-side logic via the synchronous API, gRPC provides a mocked Stub based on googletest(googlemock) that can be programmed upon and easily incorporated in the test code. - -For instance, consider an EchoService like this: - - -```proto -service EchoTestService { -        rpc Echo(EchoRequest) returns (EchoResponse); -        rpc BidiStream(stream EchoRequest) returns (stream EchoResponse); -} -``` - -The code generated would look something like this: - -```c++ -class EchoTestService final { - public: - class StubInterface { - virtual ::grpc::Status Echo(::grpc::ClientContext* context, const ::grpc::testing::EchoRequest& request, ::grpc::testing::EchoResponse* response) = 0; - … - std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>> BidiStream(::grpc::ClientContext* context) { - return std::unique_ptr< ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>>(BidiStreamRaw(context)); - } - … - private: - virtual ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>* BidiStreamRaw(::grpc::ClientContext* context) = 0; - … -  } // End StubInterface -… -} // End EchoTestService -``` - - -If we mock the StubInterface and set expectations on the pure-virtual methods we can test client-side logic without having to make any rpcs. - -A mock for this StubInterface will look like this: - - -```c++ -class MockEchoTestServiceStub : public EchoTestService::StubInterface { - public: - MOCK_METHOD3(Echo, ::grpc::Status(::grpc::ClientContext* context, const ::grpc::testing::EchoRequest& request, ::grpc::testing::EchoResponse* response)); - MOCK_METHOD1(BidiStreamRaw, ::grpc::ClientReaderWriterInterface< ::grpc::testing::EchoRequest, ::grpc::testing::EchoResponse>*(::grpc::ClientContext* context)); -}; -``` - - -**Generating mock code:** - -Such a mock can be auto-generated by: - - - -1. Setting flag(generate_mock_code=true) on grpc plugin for protoc, or -1. Setting an attribute(generate_mock) in your bazel rule. - -Protoc plugin flag: - -```sh -protoc -I . --grpc_out=generate_mock_code=true:. --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` echo.proto -``` - -Bazel rule: - -```py -grpc_proto_library( - name = "echo_proto", - srcs = ["echo.proto"], - generate_mock = True, -) -``` - - -By adding such a flag now a header file `echo_mock.grpc.pb.h` containing the mocked stub will also be generated. - -This header file can then be included in test files along with a gmock dependency. - -**Writing tests with mocked Stub.** - -Consider the following client a user might have: - -```c++ -class FakeClient { - public: - explicit FakeClient(EchoTestService::StubInterface* stub) : stub_(stub) {} - - void DoEcho() { - ClientContext context; - EchoRequest request; - EchoResponse response; - request.set_message("hello world"); - Status s = stub_->Echo(&context, request, &response); - EXPECT_EQ(request.message(), response.message()); - EXPECT_TRUE(s.ok()); - } - - void DoBidiStream() { - EchoRequest request; - EchoResponse response; - ClientContext context; - grpc::string msg("hello"); - - std::unique_ptr> - stream = stub_->BidiStream(&context); - - request.set_message(msg + "0"); - EXPECT_TRUE(stream->Write(request)); - EXPECT_TRUE(stream->Read(&response)); - EXPECT_EQ(response.message(), request.message()); - - request.set_message(msg + "1"); - EXPECT_TRUE(stream->Write(request)); - EXPECT_TRUE(stream->Read(&response)); - EXPECT_EQ(response.message(), request.message()); - - request.set_message(msg + "2"); - EXPECT_TRUE(stream->Write(request)); - EXPECT_TRUE(stream->Read(&response)); - EXPECT_EQ(response.message(), request.message()); - - stream->WritesDone(); - EXPECT_FALSE(stream->Read(&response)); - - Status s = stream->Finish(); - EXPECT_TRUE(s.ok()); - } - - void ResetStub(EchoTestService::StubInterface* stub) { stub_ = stub; } - - private: - EchoTestService::StubInterface* stub_; -}; -``` - -A test could initialize this FakeClient with a mocked stub having set expectations on it: - -Unary RPC: - -```c++ -MockEchoTestServiceStub stub; -EchoResponse resp; -resp.set_message("hello world"); -Expect_CALL(stub, Echo(_,_,_)).Times(Atleast(1)).WillOnce(DoAll(SetArgPointee<2>(resp), Return(Status::OK))); -FakeClient client(stub); -client.DoEcho(); -``` - -Streaming RPC: - -```c++ -ACTION_P(copy, msg) { - arg0->set_message(msg->message()); -} - - -auto rw = new MockClientReaderWriter(); -EchoRequest msg; -EXPECT_CALL(*rw, Write(_, _)).Times(3).WillRepeatedly(DoAll(SaveArg<0>(&msg), Return(true))); -EXPECT_CALL(*rw, Read(_)). - WillOnce(DoAll(WithArg<0>(copy(&msg)), Return(true))). - WillOnce(DoAll(WithArg<0>(copy(&msg)), Return(true))). - WillOnce(DoAll(WithArg<0>(copy(&msg)), Return(true))). - WillOnce(Return(false)); - -MockEchoTestServiceStub stub; -EXPECT_CALL(stub, BidiStreamRaw(_)).Times(AtLeast(1)).WillOnce(Return(rw)); - -FakeClient client(stub); -client.DoBidiStream(); -``` - From 6974763268e131addec3e64f25b6758f34e76c98 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 2 May 2017 14:47:06 -0700 Subject: [PATCH 173/177] Stop checking for pre-c++11 compilers since these are no longer supported --- Makefile | 8 ------ templates/Makefile.template | 8 ------ test/build/c++11.cc | 52 ------------------------------------- 3 files changed, 68 deletions(-) delete mode 100644 test/build/c++11.cc diff --git a/Makefile b/Makefile index 1a8ee553ca3..b28c0fb9e14 100644 --- a/Makefile +++ b/Makefile @@ -308,10 +308,6 @@ else TMPOUT = `mktemp /tmp/test-out-XXXXXX` endif -# Detect if we can use C++11 -CXX11_CHECK_CMD = $(CXX) -std=c++11 -o $(TMPOUT) -c test/build/c++11.cc -HAS_CXX11 = $(shell $(CXX11_CHECK_CMD) 2> /dev/null && echo true || echo false) - CHECK_SHADOW_WORKS_CMD = $(CC) -std=c99 -Werror -Wshadow -o $(TMPOUT) -c test/build/shadow.c HAS_WORKING_SHADOW = $(shell $(CHECK_SHADOW_WORKS_CMD) 2> /dev/null && echo true || echo false) ifeq ($(HAS_WORKING_SHADOW),true) @@ -342,11 +338,7 @@ HOST_LD ?= $(LD) HOST_LDXX ?= $(LDXX) CFLAGS += -std=c99 -Wsign-conversion -Wconversion $(W_SHADOW) $(W_EXTRA_SEMI) -ifeq ($(HAS_CXX11),true) CXXFLAGS += -std=c++11 -else -CXXFLAGS += -std=c++0x -endif CPPFLAGS += -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter -DOSATOMIC_USE_INLINED=1 LDFLAGS += -g diff --git a/templates/Makefile.template b/templates/Makefile.template index 5ce606f8280..a9dd171b841 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -206,10 +206,6 @@ TMPOUT = `mktemp /tmp/test-out-XXXXXX` endif - # Detect if we can use C++11 - CXX11_CHECK_CMD = $(CXX) -std=c++11 -o $(TMPOUT) -c test/build/c++11.cc - HAS_CXX11 = $(shell $(CXX11_CHECK_CMD) 2> /dev/null && echo true || echo false) - %for warning in CHECK_WARNINGS: ${warning_var('CHECK_%s_WORKS_CMD', warning)} = $(CC) -std=c99 -Werror -W${warning} -o $(TMPOUT) -c test/build/${warning}.c ${warning_var('HAS_WORKING_%s', warning)} = $(shell $(${warning_var('CHECK_%s_WORKS_CMD', warning)}) 2> /dev/null && echo true || echo false) @@ -230,11 +226,7 @@ HOST_LDXX ?= $(LDXX) CFLAGS += -std=c99 -Wsign-conversion -Wconversion ${' '.join(warning_var('$(W_%s)', warning) for warning in PREFERRED_WARNINGS)} - ifeq ($(HAS_CXX11),true) CXXFLAGS += -std=c++11 - else - CXXFLAGS += -std=c++0x - endif % for arg in ['CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS', 'DEFINES']: % if defaults.get('global', []).get(arg, None) is not None: ${arg} += ${defaults.get('global').get(arg)} diff --git a/test/build/c++11.cc b/test/build/c++11.cc deleted file mode 100644 index 4822a20e7f2..00000000000 --- a/test/build/c++11.cc +++ /dev/null @@ -1,52 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* This is just a compilation test, to see if we have C++11. */ - -#include -#include - -class Base { - public: - virtual void foo() = 0; -}; - -class Foo final : public Base { - public: - void foo() override {} -}; - -int main() { - Foo().foo(); - return 0; -} From 598fb573361599329b81de5dbb6f6a80169c18f2 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Tue, 9 May 2017 17:03:43 -0700 Subject: [PATCH 174/177] Enable IPv6 in Docker for internal CI --- .../helper_scripts/pre_build_linux.sh | 40 +++++++++++++++++++ .../linux/grpc_interop_badserver_java.sh | 3 +- .../linux/grpc_interop_badserver_python.sh | 3 +- .../internal_ci/linux/grpc_interop_tocloud.sh | 3 +- tools/internal_ci/linux/grpc_master.sh | 5 +-- tools/internal_ci/linux/grpc_portability.sh | 5 +-- .../linux/grpc_portability_build_only.sh | 5 +-- tools/internal_ci/linux/grpc_sanity.sh | 5 +-- .../linux/sanitizer/grpc_c_asan.sh | 5 +-- .../linux/sanitizer/grpc_c_msan.sh | 5 +-- .../linux/sanitizer/grpc_c_tsan.sh | 5 +-- .../linux/sanitizer/grpc_cpp_asan.sh | 5 +-- .../linux/sanitizer/grpc_cpp_tsan.sh | 5 +-- .../dockerize/build_docker_and_run_tests.sh | 1 + 14 files changed, 53 insertions(+), 42 deletions(-) create mode 100755 tools/internal_ci/helper_scripts/pre_build_linux.sh diff --git a/tools/internal_ci/helper_scripts/pre_build_linux.sh b/tools/internal_ci/helper_scripts/pre_build_linux.sh new file mode 100755 index 00000000000..5ce8fe6de36 --- /dev/null +++ b/tools/internal_ci/helper_scripts/pre_build_linux.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +# Enable IPv6 in Docker +sudo sed -i s/DOCKER_OPTS=/DOCKER_OPTS=\"--ipv6\"/g /etc/init.d/docker +sudo /etc/init.d/docker restart + +# Download Docker images from DockerHub +export DOCKERHUB_ORGANIZATION=grpctesting + +git submodule update --init diff --git a/tools/internal_ci/linux/grpc_interop_badserver_java.sh b/tools/internal_ci/linux/grpc_interop_badserver_java.sh index c309c623e0b..e85949bbff0 100755 --- a/tools/internal_ci/linux/grpc_interop_badserver_java.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_java.sh @@ -35,7 +35,6 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../../.. -git submodule update --init - +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_interop_tests.py -l java --use_docker --http2_server_interop $@ diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.sh b/tools/internal_ci/linux/grpc_interop_badserver_python.sh index c3bb92f33da..2010597d91b 100755 --- a/tools/internal_ci/linux/grpc_interop_badserver_python.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_python.sh @@ -35,7 +35,6 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../../.. -git submodule update --init - +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_interop_tests.py -l python --use_docker --http2_server_interop $@ diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.sh b/tools/internal_ci/linux/grpc_interop_tocloud.sh index 572001d9441..5f3c603a131 100755 --- a/tools/internal_ci/linux/grpc_interop_tocloud.sh +++ b/tools/internal_ci/linux/grpc_interop_tocloud.sh @@ -35,6 +35,5 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../../.. -git submodule update --init - +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_interop_tests.py -l all -s all --use_docker --http2_interop -t -j 12 $@ diff --git a/tools/internal_ci/linux/grpc_master.sh b/tools/internal_ci/linux/grpc_master.sh index bd22dd069c2..b7184c3d7db 100755 --- a/tools/internal_ci/linux/grpc_master.sh +++ b/tools/internal_ci/linux/grpc_master.sh @@ -43,8 +43,5 @@ docker --version || true # Need to increase open files limit for c tests ulimit -n 32768 -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f basictests linux --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh index 6c55ed71cbb..bc7e8ca3579 100755 --- a/tools/internal_ci/linux/grpc_portability.sh +++ b/tools/internal_ci/linux/grpc_portability.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f portability linux --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh index 787f0302c08..fda511ea673 100755 --- a/tools/internal_ci/linux/grpc_portability_build_only.sh +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f portability linux --internal_ci --build_only diff --git a/tools/internal_ci/linux/grpc_sanity.sh b/tools/internal_ci/linux/grpc_sanity.sh index fac25c75315..2c6712a893a 100755 --- a/tools/internal_ci/linux/grpc_sanity.sh +++ b/tools/internal_ci/linux/grpc_sanity.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -git submodule update --init - -# download base docker image from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests.py -l sanity -c opt -t -x sponge_log.xml --use_docker --report_suite_name sanity_linux_opt diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh b/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh index 2927ad7de48..be5ad1c1ae2 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f c asan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh b/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh index 3a3c850a9f2..299f7739fe3 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f c msan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh index daebf34521e..6429d07a638 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f c tsan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh index 29f7fda74b1..1303d4a110b 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f c++ asan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh index 4b9291a4a00..972e33f542d 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh @@ -33,8 +33,5 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -git submodule update --init - -# download docker images from dockerhub -export DOCKERHUB_ORGANIZATION=grpctesting +tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f c++ tsan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh index 731bb02d214..79287454c93 100755 --- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh +++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh @@ -82,6 +82,7 @@ docker run \ -e "BUILD_URL=$BUILD_URL" \ -e "JOB_BASE_NAME=$JOB_BASE_NAME" \ -i $TTY_FLAG \ + --sysctl net.ipv6.conf.all.disable_ipv6=0 \ -v ~/.config/gcloud:/root/.config/gcloud \ -v "$git_root:$docker_instance_git_root" \ -v /tmp/ccache:/tmp/ccache \ From e44afe7170a1d40b91054f7bfc51c5ce523e5f51 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 11 May 2017 13:02:13 +0200 Subject: [PATCH 175/177] source the linux rc file --- .../{pre_build_linux.sh => prepare_build_linux_rc} | 7 +++---- .../internal_ci/linux/grpc_interop_badserver_java.sh | 3 ++- .../linux/grpc_interop_badserver_python.sh | 3 ++- tools/internal_ci/linux/grpc_interop_tocloud.sh | 3 ++- tools/internal_ci/linux/grpc_master.sh | 11 +---------- tools/internal_ci/linux/grpc_portability.sh | 3 ++- .../internal_ci/linux/grpc_portability_build_only.sh | 3 ++- tools/internal_ci/linux/grpc_sanity.sh | 3 ++- tools/internal_ci/linux/sanitizer/grpc_c_asan.sh | 3 ++- tools/internal_ci/linux/sanitizer/grpc_c_msan.sh | 3 ++- tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh | 3 ++- tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh | 3 ++- tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh | 3 ++- 13 files changed, 26 insertions(+), 25 deletions(-) rename tools/internal_ci/helper_scripts/{pre_build_linux.sh => prepare_build_linux_rc} (92%) mode change 100755 => 100644 diff --git a/tools/internal_ci/helper_scripts/pre_build_linux.sh b/tools/internal_ci/helper_scripts/prepare_build_linux_rc old mode 100755 new mode 100644 similarity index 92% rename from tools/internal_ci/helper_scripts/pre_build_linux.sh rename to tools/internal_ci/helper_scripts/prepare_build_linux_rc index 5ce8fe6de36..c8cb5a0c40f --- a/tools/internal_ci/helper_scripts/pre_build_linux.sh +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -28,11 +28,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set -ex +# Source this rc script to prepare the environment for linux builds -# Enable IPv6 in Docker -sudo sed -i s/DOCKER_OPTS=/DOCKER_OPTS=\"--ipv6\"/g /etc/init.d/docker -sudo /etc/init.d/docker restart +# Need to increase open files limit for c tests +ulimit -n 32768 # Download Docker images from DockerHub export DOCKERHUB_ORGANIZATION=grpctesting diff --git a/tools/internal_ci/linux/grpc_interop_badserver_java.sh b/tools/internal_ci/linux/grpc_interop_badserver_java.sh index e85949bbff0..02d7b9d4316 100755 --- a/tools/internal_ci/linux/grpc_interop_badserver_java.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_java.sh @@ -35,6 +35,7 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_interop_tests.py -l java --use_docker --http2_server_interop $@ diff --git a/tools/internal_ci/linux/grpc_interop_badserver_python.sh b/tools/internal_ci/linux/grpc_interop_badserver_python.sh index 2010597d91b..3ceb181d904 100755 --- a/tools/internal_ci/linux/grpc_interop_badserver_python.sh +++ b/tools/internal_ci/linux/grpc_interop_badserver_python.sh @@ -35,6 +35,7 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_interop_tests.py -l python --use_docker --http2_server_interop $@ diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.sh b/tools/internal_ci/linux/grpc_interop_tocloud.sh index 5f3c603a131..a3067e70e67 100755 --- a/tools/internal_ci/linux/grpc_interop_tocloud.sh +++ b/tools/internal_ci/linux/grpc_interop_tocloud.sh @@ -35,5 +35,6 @@ export LANG=en_US.UTF-8 # Enter the gRPC repo root cd $(dirname $0)/../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_interop_tests.py -l all -s all --use_docker --http2_interop -t -j 12 $@ diff --git a/tools/internal_ci/linux/grpc_master.sh b/tools/internal_ci/linux/grpc_master.sh index b7184c3d7db..d3c89bfa071 100755 --- a/tools/internal_ci/linux/grpc_master.sh +++ b/tools/internal_ci/linux/grpc_master.sh @@ -33,15 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -# TODO(jtattermusch): get rid of the system inspection eventually -nproc || true -lsb_release -dc || true -gcc --version || true -clang --version || true -docker --version || true +source tools/internal_ci/helper_scripts/prepare_build_linux_rc -# Need to increase open files limit for c tests -ulimit -n 32768 - -tools/internal_ci/helper_scripts/pre_build_linux.sh tools/run_tests/run_tests_matrix.py -f basictests linux --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/grpc_portability.sh b/tools/internal_ci/linux/grpc_portability.sh index bc7e8ca3579..64959c793f3 100755 --- a/tools/internal_ci/linux/grpc_portability.sh +++ b/tools/internal_ci/linux/grpc_portability.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f portability linux --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/grpc_portability_build_only.sh b/tools/internal_ci/linux/grpc_portability_build_only.sh index fda511ea673..099c3f8948c 100755 --- a/tools/internal_ci/linux/grpc_portability_build_only.sh +++ b/tools/internal_ci/linux/grpc_portability_build_only.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f portability linux --internal_ci --build_only diff --git a/tools/internal_ci/linux/grpc_sanity.sh b/tools/internal_ci/linux/grpc_sanity.sh index 2c6712a893a..7166ce7d248 100755 --- a/tools/internal_ci/linux/grpc_sanity.sh +++ b/tools/internal_ci/linux/grpc_sanity.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests.py -l sanity -c opt -t -x sponge_log.xml --use_docker --report_suite_name sanity_linux_opt diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh b/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh index be5ad1c1ae2..5a61d9d5d1b 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_c_asan.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f c asan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh b/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh index 299f7739fe3..1c3b90dce22 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_c_msan.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f c msan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh index 6429d07a638..495a004c9df 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f c tsan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh index 1303d4a110b..99219e35159 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f c++ asan --inner_jobs 16 -j 1 --internal_ci diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh index 972e33f542d..be46af902c5 100755 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.sh @@ -33,5 +33,6 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../../.. -tools/internal_ci/helper_scripts/pre_build_linux.sh +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + tools/run_tests/run_tests_matrix.py -f c++ tsan --inner_jobs 16 -j 1 --internal_ci From 822aae53d37bd569780b88bae5caa1fa2a5987ca Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 11 May 2017 10:54:22 -0700 Subject: [PATCH 176/177] Always receive initial metadata in this test --- test/core/end2end/tests/cancel_after_invoke.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/core/end2end/tests/cancel_after_invoke.c b/test/core/end2end/tests/cancel_after_invoke.c index 5bc9ed283bb..6deb86ea3e4 100644 --- a/test/core/end2end/tests/cancel_after_invoke.c +++ b/test/core/end2end/tests/cancel_after_invoke.c @@ -147,6 +147,11 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config, op->flags = 0; op->reserved = NULL; op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; op->op = GRPC_OP_SEND_INITIAL_METADATA; op->data.send_initial_metadata.count = 0; op->flags = 0; @@ -161,11 +166,6 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config, op->flags = 0; op->reserved = NULL; op++; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; - op->flags = 0; - op->reserved = NULL; - op++; op->op = GRPC_OP_RECV_MESSAGE; op->data.recv_message.recv_message = &response_payload_recv; op->flags = 0; @@ -200,7 +200,7 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config, void cancel_after_invoke(grpc_end2end_test_config config) { unsigned i, j; - for (j = 2; j < 6; j++) { + for (j = 3; j < 6; j++) { for (i = 0; i < GPR_ARRAY_SIZE(cancellation_modes); i++) { test_cancel_after_invoke(config, cancellation_modes[i], j); } From 3ca8134514536be3014d692388d6e329025cd25e Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Wed, 10 May 2017 12:35:23 -0700 Subject: [PATCH 177/177] Expose Auth Context in Python --- src/python/grpcio/grpc/__init__.py | 36 ++++ .../grpcio/grpc/_cython/_cygrpc/grpc.pxi | 29 ++++ .../grpc/_cython/_cygrpc/security.pyx.pxi | 58 +++++++ src/python/grpcio/grpc/_server.py | 15 ++ src/python/grpcio_tests/tests/tests.json | 1 + .../tests/unit/_auth_context_test.py | 154 ++++++++++++++++++ 6 files changed, 293 insertions(+) create mode 100644 src/python/grpcio_tests/tests/unit/_auth_context_test.py diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index 2952dcd36a9..5426b47c761 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -776,6 +776,42 @@ class ServicerContext(six.with_metaclass(abc.ABCMeta, RpcContext)): """ raise NotImplementedError() + @abc.abstractmethod + def peer_identities(self): + """Gets one or more peer identity(s). + + Equivalent to + servicer_context.auth_context().get( + servicer_context.peer_identity_key()) + + Returns: + An iterable of the identities, or None if the call is not authenticated. + Each identity is returned as a raw bytes type. + """ + raise NotImplementedError() + + @abc.abstractmethod + def peer_identity_key(self): + """The auth property used to identify the peer. + + For example, "x509_common_name" or "x509_subject_alternative_name" are + used to identify an SSL peer. + + Returns: + The auth property (string) that indicates the + peer identity, or None if the call is not authenticated. + """ + raise NotImplementedError() + + @abc.abstractmethod + def auth_context(self): + """Gets the auth context for the call. + + Returns: + A map of strings to an iterable of bytes for each auth property. + """ + raise NotImplementedError() + @abc.abstractmethod def send_initial_metadata(self, initial_metadata): """Sends the initial metadata value to the client. diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 1db2056d476..e71d3e7dc1c 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -486,6 +486,35 @@ cdef extern from "grpc/grpc_security.h": grpc_call_credentials *grpc_metadata_credentials_create_from_plugin( grpc_metadata_credentials_plugin plugin, void *reserved) nogil + ctypedef struct grpc_auth_property_iterator: + pass + + ctypedef struct grpc_auth_property: + char *name + char *value + size_t value_length + + grpc_auth_property *grpc_auth_property_iterator_next( + grpc_auth_property_iterator *it) + + grpc_auth_property_iterator grpc_auth_context_property_iterator( + const grpc_auth_context *ctx) + + grpc_auth_property_iterator grpc_auth_context_peer_identity( + const grpc_auth_context *ctx) + + char *grpc_auth_context_peer_identity_property_name( + const grpc_auth_context *ctx) + + grpc_auth_property_iterator grpc_auth_context_find_properties_by_name( + const grpc_auth_context *ctx, const char *name) + + grpc_auth_context_peer_is_authenticated( + const grpc_auth_context *ctx) + + grpc_auth_context *grpc_call_auth_context(grpc_call *call) + + void grpc_auth_context_release(grpc_auth_context *context) cdef extern from "grpc/compression.h": diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi index 357b0330d5c..a21eac79951 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi @@ -44,3 +44,61 @@ cdef grpc_ssl_roots_override_result ssl_roots_override_callback( pem_root_certs[0][len(temporary_pem_root_certs)] = '\0' return GRPC_SSL_ROOTS_OVERRIDE_OK + + +def peer_identities(Call call): + cdef grpc_auth_context* auth_context + cdef grpc_auth_property_iterator properties + cdef grpc_auth_property* property + + auth_context = grpc_call_auth_context(call.c_call) + if auth_context == NULL: + return None + properties = grpc_auth_context_peer_identity(auth_context) + identities = [] + while True: + property = grpc_auth_property_iterator_next(&properties) + if property == NULL: + break + if property.value != NULL: + identities.append((property.value)) + grpc_auth_context_release(auth_context) + return identities if identities else None + +def peer_identity_key(Call call): + cdef grpc_auth_context* auth_context + cdef char* c_key + auth_context = grpc_call_auth_context(call.c_call) + if auth_context == NULL: + return None + c_key = grpc_auth_context_peer_identity_property_name(auth_context) + if c_key == NULL: + key = None + else: + key = grpc_auth_context_peer_identity_property_name(auth_context) + grpc_auth_context_release(auth_context) + return key + +def auth_context(Call call): + cdef grpc_auth_context* auth_context + cdef grpc_auth_property_iterator properties + cdef grpc_auth_property* property + + auth_context = grpc_call_auth_context(call.c_call) + if auth_context == NULL: + return {} + properties = grpc_auth_context_property_iterator(auth_context) + py_auth_context = {} + while True: + property = grpc_auth_property_iterator_next(&properties) + if property == NULL: + break + if property.name != NULL and property.value != NULL: + key = property.name + if key in py_auth_context: + py_auth_context[key].append((property.value)) + else: + py_auth_context[key] = [ property.value] + grpc_auth_context_release(auth_context) + return py_auth_context + diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index f29c44a4cfd..860085f0c7e 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -31,6 +31,7 @@ import collections import enum import logging +import six import threading import time @@ -255,6 +256,20 @@ class _Context(grpc.ServicerContext): def peer(self): return _common.decode(self._rpc_event.operation_call.peer()) + def peer_identities(self): + return cygrpc.peer_identities(self._rpc_event.operation_call) + + def peer_identity_key(self): + id_key = cygrpc.peer_identity_key(self._rpc_event.operation_call) + return id_key if id_key is None else _common.decode(id_key) + + def auth_context(self): + return { + _common.decode(key): value + for key, value in six.iteritems( + cygrpc.auth_context(self._rpc_event.operation_call)) + } + def send_initial_metadata(self, initial_metadata): with self._state.condition: if self._state.client is _CANCELLED: diff --git a/src/python/grpcio_tests/tests/tests.json b/src/python/grpcio_tests/tests/tests.json index f750b051022..9f7587faa66 100644 --- a/src/python/grpcio_tests/tests/tests.json +++ b/src/python/grpcio_tests/tests/tests.json @@ -12,6 +12,7 @@ "unit._api_test.AllTest", "unit._api_test.ChannelConnectivityTest", "unit._api_test.ChannelTest", + "unit._auth_context_test.AuthContextTest", "unit._auth_test.AccessTokenCallCredentialsTest", "unit._auth_test.GoogleCallCredentialsTest", "unit._channel_args_test.ChannelArgsTest", diff --git a/src/python/grpcio_tests/tests/unit/_auth_context_test.py b/src/python/grpcio_tests/tests/unit/_auth_context_test.py new file mode 100644 index 00000000000..fce6ac967db --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/_auth_context_test.py @@ -0,0 +1,154 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Tests exposure of SSL auth context""" + +import pickle +import unittest + +import grpc +from grpc import _channel +from grpc.framework.foundation import logging_pool +import six + +from tests.unit import test_common +from tests.unit.framework.common import test_constants +from tests.unit import resources + +_REQUEST = b'\x00\x00\x00' +_RESPONSE = b'\x00\x00\x00' + +_UNARY_UNARY = '/test/UnaryUnary' + +_SERVER_HOST_OVERRIDE = 'foo.test.google.fr' +_CLIENT_IDS = (b'*.test.google.fr', b'waterzooi.test.google.be', + b'*.test.youtube.com', b'192.168.1.3',) +_ID = 'id' +_ID_KEY = 'id_key' +_AUTH_CTX = 'auth_ctx' + +_PRIVATE_KEY = resources.private_key() +_CERTIFICATE_CHAIN = resources.certificate_chain() +_TEST_ROOT_CERTIFICATES = resources.test_root_certificates() +_SERVER_CERTS = ((_PRIVATE_KEY, _CERTIFICATE_CHAIN),) +_PROPERTY_OPTIONS = (('grpc.ssl_target_name_override', _SERVER_HOST_OVERRIDE,),) + + +def handle_unary_unary(request, servicer_context): + return pickle.dumps({ + _ID: servicer_context.peer_identities(), + _ID_KEY: servicer_context.peer_identity_key(), + _AUTH_CTX: servicer_context.auth_context() + }) + + +class AuthContextTest(unittest.TestCase): + + def testInsecure(self): + server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) + handler = grpc.method_handlers_generic_handler('test', { + 'UnaryUnary': + grpc.unary_unary_rpc_method_handler(handle_unary_unary) + }) + server = grpc.server(server_pool, (handler,)) + port = server.add_insecure_port('[::]:0') + server.start() + + channel = grpc.insecure_channel('localhost:%d' % port) + response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + server.stop(None) + + auth_data = pickle.loads(response) + self.assertIsNone(auth_data[_ID]) + self.assertIsNone(auth_data[_ID_KEY]) + self.assertDictEqual({}, auth_data[_AUTH_CTX]) + + def testSecureNoCert(self): + server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) + handler = grpc.method_handlers_generic_handler('test', { + 'UnaryUnary': + grpc.unary_unary_rpc_method_handler(handle_unary_unary) + }) + server = grpc.server(server_pool, (handler,)) + server_cred = grpc.ssl_server_credentials(_SERVER_CERTS) + port = server.add_secure_port('[::]:0', server_cred) + server.start() + + channel_creds = grpc.ssl_channel_credentials( + root_certificates=_TEST_ROOT_CERTIFICATES) + channel = grpc.secure_channel( + 'localhost:{}'.format(port), + channel_creds, + options=_PROPERTY_OPTIONS) + response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + server.stop(None) + + auth_data = pickle.loads(response) + self.assertIsNone(auth_data[_ID]) + self.assertIsNone(auth_data[_ID_KEY]) + self.assertDictEqual({ + 'transport_security_type': [b'ssl'] + }, auth_data[_AUTH_CTX]) + + def testSecureClientCert(self): + server_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY) + handler = grpc.method_handlers_generic_handler('test', { + 'UnaryUnary': + grpc.unary_unary_rpc_method_handler(handle_unary_unary) + }) + server = grpc.server(server_pool, (handler,)) + server_cred = grpc.ssl_server_credentials( + _SERVER_CERTS, + root_certificates=_TEST_ROOT_CERTIFICATES, + require_client_auth=True) + port = server.add_secure_port('[::]:0', server_cred) + server.start() + + channel_creds = grpc.ssl_channel_credentials( + root_certificates=_TEST_ROOT_CERTIFICATES, + private_key=_PRIVATE_KEY, + certificate_chain=_CERTIFICATE_CHAIN) + channel = grpc.secure_channel( + 'localhost:{}'.format(port), + channel_creds, + options=_PROPERTY_OPTIONS) + + response = channel.unary_unary(_UNARY_UNARY)(_REQUEST) + server.stop(None) + + auth_data = pickle.loads(response) + auth_ctx = auth_data[_AUTH_CTX] + six.assertCountEqual(self, _CLIENT_IDS, auth_data[_ID]) + self.assertEqual('x509_subject_alternative_name', auth_data[_ID_KEY]) + self.assertSequenceEqual([b'ssl'], auth_ctx['transport_security_type']) + self.assertSequenceEqual([b'*.test.google.com'], + auth_ctx['x509_common_name']) + + +if __name__ == '__main__': + unittest.main(verbosity=2)