|
|
|
@ -49,7 +49,8 @@ |
|
|
|
|
#include "src/core/lib/support/block_annotate.h" |
|
|
|
|
#include "src/core/lib/support/spinlock.h" |
|
|
|
|
|
|
|
|
|
// debug aid: create workers on the heap (allows asan to spot use-after-destruction)
|
|
|
|
|
// debug aid: create workers on the heap (allows asan to spot
|
|
|
|
|
// use-after-destruction)
|
|
|
|
|
#define GRPC_EPOLLEX_CREATE_WORKERS_ON_HEAP 1 |
|
|
|
|
|
|
|
|
|
#ifndef NDEBUG |
|
|
|
@ -461,7 +462,8 @@ static pollable *pollable_ref(pollable *p) { |
|
|
|
|
static pollable *pollable_ref(pollable *p, int line, const char *reason) { |
|
|
|
|
if (GRPC_TRACER_ON(grpc_trace_pollable_refcount)) { |
|
|
|
|
int r = (int)gpr_atm_no_barrier_load(&p->refs.count); |
|
|
|
|
gpr_log(__FILE__, line, GPR_LOG_SEVERITY_DEBUG, "POLLABLE:%p ref %d->%d %s", p, r, r+1, reason); |
|
|
|
|
gpr_log(__FILE__, line, GPR_LOG_SEVERITY_DEBUG, |
|
|
|
|
"POLLABLE:%p ref %d->%d %s", p, r, r + 1, reason); |
|
|
|
|
} |
|
|
|
|
#endif |
|
|
|
|
gpr_ref(&p->refs); |
|
|
|
@ -475,7 +477,8 @@ static void pollable_unref(pollable *p, int line, const char *reason) { |
|
|
|
|
if (p == NULL) return; |
|
|
|
|
if (GRPC_TRACER_ON(grpc_trace_pollable_refcount)) { |
|
|
|
|
int r = (int)gpr_atm_no_barrier_load(&p->refs.count); |
|
|
|
|
gpr_log(__FILE__, line, GPR_LOG_SEVERITY_DEBUG, "POLLABLE:%p unref %d->%d %s", p, r, r-1, reason); |
|
|
|
|
gpr_log(__FILE__, line, GPR_LOG_SEVERITY_DEBUG, |
|
|
|
|
"POLLABLE:%p unref %d->%d %s", p, r, r - 1, reason); |
|
|
|
|
} |
|
|
|
|
#endif |
|
|
|
|
if (p != NULL && gpr_unref(&p->refs)) { |
|
|
|
@ -581,8 +584,10 @@ static grpc_error *pollset_kick_inner(grpc_exec_ctx *exec_ctx, |
|
|
|
|
if (GRPC_TRACER_ON(grpc_polling_trace)) { |
|
|
|
|
gpr_log(GPR_DEBUG, |
|
|
|
|
"PS:%p kick %p tls_pollset=%p tls_worker=%p pollset.root_worker=%p", |
|
|
|
|
pollset, specific_worker, (void *)gpr_tls_get(&g_current_thread_pollset), |
|
|
|
|
(void *)gpr_tls_get(&g_current_thread_worker), pollset->root_worker); |
|
|
|
|
pollset, specific_worker, |
|
|
|
|
(void *)gpr_tls_get(&g_current_thread_pollset), |
|
|
|
|
(void *)gpr_tls_get(&g_current_thread_worker), |
|
|
|
|
pollset->root_worker); |
|
|
|
|
} |
|
|
|
|
if (specific_worker == NULL) { |
|
|
|
|
if (gpr_tls_get(&g_current_thread_pollset) != (intptr_t)pollset) { |
|
|
|
@ -850,7 +855,8 @@ static bool begin_worker(grpc_pollset *pollset, grpc_pollset_worker *worker, |
|
|
|
|
worker->initialized_cv = false; |
|
|
|
|
worker->kicked = false; |
|
|
|
|
worker->pollset = pollset; |
|
|
|
|
worker->pollable_obj = POLLABLE_REF(pollset->active_pollable, "pollset_worker"); |
|
|
|
|
worker->pollable_obj = |
|
|
|
|
POLLABLE_REF(pollset->active_pollable, "pollset_worker"); |
|
|
|
|
worker_insert(&pollset->root_worker, worker, PWLINK_POLLSET); |
|
|
|
|
gpr_mu_lock(&worker->pollable_obj->mu); |
|
|
|
|
if (!worker_insert(&worker->pollable_obj->root_worker, worker, |
|
|
|
@ -930,7 +936,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
gpr_log(GPR_DEBUG, "PS:%p work hdl=%p worker=%p now=%" PRId64 |
|
|
|
|
".%09d deadline=%" PRId64 ".%09d kwp=%d pollable=%p", |
|
|
|
|
pollset, worker_hdl, WORKER_PTR, now.tv_sec, now.tv_nsec, |
|
|
|
|
deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, pollset->active_pollable); |
|
|
|
|
deadline.tv_sec, deadline.tv_nsec, pollset->kicked_without_poller, |
|
|
|
|
pollset->active_pollable); |
|
|
|
|
} |
|
|
|
|
static const char *err_desc = "pollset_work"; |
|
|
|
|
grpc_error *error = GRPC_ERROR_NONE; |
|
|
|
@ -943,7 +950,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
GPR_ASSERT(!pollset->shutdown_closure); |
|
|
|
|
gpr_mu_unlock(&pollset->mu); |
|
|
|
|
if (pollset->event_cursor == pollset->event_count) { |
|
|
|
|
append_error(&error, pollset_epoll(exec_ctx, pollset, WORKER_PTR->pollable_obj, |
|
|
|
|
append_error(&error, |
|
|
|
|
pollset_epoll(exec_ctx, pollset, WORKER_PTR->pollable_obj, |
|
|
|
|
now, deadline), |
|
|
|
|
err_desc); |
|
|
|
|
} |
|
|
|
@ -967,7 +975,8 @@ static grpc_error *pollset_transition_pollable_from_empty_to_fd_locked( |
|
|
|
|
static const char *err_desc = "pollset_transition_pollable_from_empty_to_fd"; |
|
|
|
|
grpc_error *error = GRPC_ERROR_NONE; |
|
|
|
|
if (GRPC_TRACER_ON(grpc_polling_trace)) { |
|
|
|
|
gpr_log(GPR_DEBUG, "PS:%p add fd %p (%d); transition pollable from empty to fd", |
|
|
|
|
gpr_log(GPR_DEBUG, |
|
|
|
|
"PS:%p add fd %p (%d); transition pollable from empty to fd", |
|
|
|
|
pollset, fd, fd->fd); |
|
|
|
|
} |
|
|
|
|
append_error(&error, pollset_kick_all(exec_ctx, pollset), err_desc); |
|
|
|
@ -982,9 +991,11 @@ static grpc_error *pollset_transition_pollable_from_fd_to_multi_locked( |
|
|
|
|
static const char *err_desc = "pollset_transition_pollable_from_fd_to_multi"; |
|
|
|
|
grpc_error *error = GRPC_ERROR_NONE; |
|
|
|
|
if (GRPC_TRACER_ON(grpc_polling_trace)) { |
|
|
|
|
gpr_log(GPR_DEBUG, |
|
|
|
|
gpr_log( |
|
|
|
|
GPR_DEBUG, |
|
|
|
|
"PS:%p add fd %p (%d); transition pollable from fd %p to multipoller", |
|
|
|
|
pollset, and_add_fd, and_add_fd?and_add_fd->fd:-1, pollset->active_pollable->owner_fd); |
|
|
|
|
pollset, and_add_fd, and_add_fd ? and_add_fd->fd : -1, |
|
|
|
|
pollset->active_pollable->owner_fd); |
|
|
|
|
} |
|
|
|
|
append_error(&error, pollset_kick_all(exec_ctx, pollset), err_desc); |
|
|
|
|
grpc_fd *initial_fd = pollset->active_pollable->owner_fd; |
|
|
|
@ -1007,7 +1018,8 @@ static grpc_error *pollset_transition_pollable_from_fd_to_multi_locked( |
|
|
|
|
static grpc_error *pollset_add_fd_locked(grpc_exec_ctx *exec_ctx, |
|
|
|
|
grpc_pollset *pollset, grpc_fd *fd) { |
|
|
|
|
grpc_error *error = GRPC_ERROR_NONE; |
|
|
|
|
pollable *po_at_start = POLLABLE_REF(pollset->active_pollable, "pollset_add_fd"); |
|
|
|
|
pollable *po_at_start = |
|
|
|
|
POLLABLE_REF(pollset->active_pollable, "pollset_add_fd"); |
|
|
|
|
switch (pollset->active_pollable->type) { |
|
|
|
|
case PO_EMPTY: |
|
|
|
|
/* empty pollable --> single fd pollable */ |
|
|
|
@ -1037,7 +1049,8 @@ static grpc_error *pollset_as_multipollable(grpc_exec_ctx *exec_ctx, |
|
|
|
|
pollable **pollable_obj) { |
|
|
|
|
grpc_error *error = GRPC_ERROR_NONE; |
|
|
|
|
gpr_mu_lock(&pollset->mu); |
|
|
|
|
pollable *po_at_start = POLLABLE_REF(pollset->active_pollable, "pollset_as_multipollable"); |
|
|
|
|
pollable *po_at_start = |
|
|
|
|
POLLABLE_REF(pollset->active_pollable, "pollset_as_multipollable"); |
|
|
|
|
switch (pollset->active_pollable->type) { |
|
|
|
|
case PO_EMPTY: |
|
|
|
|
POLLABLE_UNREF(pollset->active_pollable, "pollset"); |
|
|
|
@ -1091,8 +1104,7 @@ static grpc_pollset_set *pollset_set_create(void) { |
|
|
|
|
return pss; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void pollset_set_unref(grpc_exec_ctx *exec_ctx, |
|
|
|
|
grpc_pollset_set *pss) { |
|
|
|
|
static void pollset_set_unref(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss) { |
|
|
|
|
if (pss == NULL) return; |
|
|
|
|
if (!gpr_unref(&pss->refs)) return; |
|
|
|
|
pollset_set_unref(exec_ctx, pss->parent); |
|
|
|
|