|
|
|
@ -54,6 +54,9 @@ |
|
|
|
|
// use-after-destruction)
|
|
|
|
|
//#define GRPC_EPOLLEX_CREATE_WORKERS_ON_HEAP 1
|
|
|
|
|
|
|
|
|
|
#define MAX_EPOLL_EVENTS 100 |
|
|
|
|
#define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 |
|
|
|
|
|
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
grpc_tracer_flag grpc_trace_pollable_refcount = |
|
|
|
|
GRPC_TRACER_INITIALIZER(false, "pollable_refcount"); |
|
|
|
@ -83,6 +86,10 @@ struct pollable { |
|
|
|
|
|
|
|
|
|
gpr_mu mu; |
|
|
|
|
grpc_pollset_worker *root_worker; |
|
|
|
|
|
|
|
|
|
int event_cursor; |
|
|
|
|
int event_count; |
|
|
|
|
struct epoll_event events[MAX_EPOLL_EVENTS]; |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
static const char *pollable_type_string(pollable_type t) { |
|
|
|
@ -174,9 +181,6 @@ struct grpc_pollset_worker { |
|
|
|
|
pwlink links[PWLINK_COUNT]; |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
#define MAX_EPOLL_EVENTS 100 |
|
|
|
|
#define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 5 |
|
|
|
|
|
|
|
|
|
struct grpc_pollset { |
|
|
|
|
gpr_mu mu; |
|
|
|
|
pollable *active_pollable; |
|
|
|
@ -184,10 +188,6 @@ struct grpc_pollset { |
|
|
|
|
grpc_closure *shutdown_closure; |
|
|
|
|
grpc_pollset_worker *root_worker; |
|
|
|
|
int containing_pollset_set_count; |
|
|
|
|
|
|
|
|
|
int event_cursor; |
|
|
|
|
int event_count; |
|
|
|
|
struct epoll_event events[MAX_EPOLL_EVENTS]; |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
/*******************************************************************************
|
|
|
|
@ -725,15 +725,15 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
pollset_maybe_finish_shutdown(exec_ctx, pollset); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, |
|
|
|
|
grpc_pollset *pollset, bool drain) { |
|
|
|
|
static grpc_error *pollable_process_events(grpc_exec_ctx *exec_ctx, grpc_pollset*pollset, |
|
|
|
|
pollable *pollable_obj, bool drain) { |
|
|
|
|
static const char *err_desc = "pollset_process_events"; |
|
|
|
|
grpc_error *error = GRPC_ERROR_NONE; |
|
|
|
|
for (int i = 0; (drain || i < MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL) && |
|
|
|
|
pollset->event_cursor != pollset->event_count; |
|
|
|
|
pollable_obj->event_cursor != pollable_obj->event_count; |
|
|
|
|
i++) { |
|
|
|
|
int n = pollset->event_cursor++; |
|
|
|
|
struct epoll_event *ev = &pollset->events[n]; |
|
|
|
|
int n = pollable_obj->event_cursor++; |
|
|
|
|
struct epoll_event *ev = &pollable_obj->events[n]; |
|
|
|
|
void *data_ptr = ev->data.ptr; |
|
|
|
|
if (1 & (intptr_t)data_ptr) { |
|
|
|
|
if (GRPC_TRACER_ON(grpc_polling_trace)) { |
|
|
|
@ -770,8 +770,6 @@ static grpc_error *pollset_process_events(grpc_exec_ctx *exec_ctx, |
|
|
|
|
static void pollset_destroy(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) { |
|
|
|
|
POLLABLE_UNREF(pollset->active_pollable, "pollset"); |
|
|
|
|
pollset->active_pollable = NULL; |
|
|
|
|
GRPC_LOG_IF_ERROR("pollset_process_events", |
|
|
|
|
pollset_process_events(exec_ctx, pollset, true)); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
@ -790,7 +788,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
int r; |
|
|
|
|
do { |
|
|
|
|
GRPC_STATS_INC_SYSCALL_POLL(exec_ctx); |
|
|
|
|
r = epoll_wait(p->epfd, pollset->events, MAX_EPOLL_EVENTS, timeout); |
|
|
|
|
r = epoll_wait(p->epfd, p->events, MAX_EPOLL_EVENTS, timeout); |
|
|
|
|
} while (r < 0 && errno == EINTR); |
|
|
|
|
if (timeout != 0) { |
|
|
|
|
GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(exec_ctx); |
|
|
|
@ -802,8 +800,8 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
gpr_log(GPR_DEBUG, "PS:%p poll %p got %d events", pollset, p, r); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
pollset->event_cursor = 0; |
|
|
|
|
pollset->event_count = r; |
|
|
|
|
p->event_cursor = 0; |
|
|
|
|
p->event_count = r; |
|
|
|
|
|
|
|
|
|
return GRPC_ERROR_NONE; |
|
|
|
|
} |
|
|
|
@ -852,7 +850,7 @@ static bool begin_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
grpc_pollset_worker *worker, |
|
|
|
|
grpc_pollset_worker **worker_hdl, |
|
|
|
|
grpc_millis deadline) { |
|
|
|
|
bool do_poll = true; |
|
|
|
|
bool do_poll = (pollset->shutdown_closure == nullptr); |
|
|
|
|
if (worker_hdl != NULL) *worker_hdl = worker; |
|
|
|
|
worker->initialized_cv = false; |
|
|
|
|
worker->kicked = false; |
|
|
|
@ -899,23 +897,33 @@ static bool begin_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
gpr_mu_unlock(&worker->pollable_obj->mu); |
|
|
|
|
|
|
|
|
|
return do_poll; |
|
|
|
|
// && pollset->shutdown_closure == NULL && pollset->active_pollable ==
|
|
|
|
|
// worker->pollable_obj;
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void end_worker(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
grpc_pollset_worker *worker, |
|
|
|
|
grpc_pollset_worker **worker_hdl) { |
|
|
|
|
gpr_mu_lock(&pollset->mu); |
|
|
|
|
gpr_mu_lock(&worker->pollable_obj->mu); |
|
|
|
|
if (worker_remove(&worker->pollable_obj->root_worker, worker, |
|
|
|
|
PWLINK_POLLABLE) == WRR_NEW_ROOT) { |
|
|
|
|
grpc_pollset_worker *new_root = worker->pollable_obj->root_worker; |
|
|
|
|
GPR_ASSERT(new_root->initialized_cv); |
|
|
|
|
gpr_cv_signal(&new_root->cv); |
|
|
|
|
switch (worker_remove(&worker->pollable_obj->root_worker, worker, |
|
|
|
|
PWLINK_POLLABLE)) { |
|
|
|
|
case WRR_NEW_ROOT: { |
|
|
|
|
// wakeup new poller
|
|
|
|
|
grpc_pollset_worker *new_root = worker->pollable_obj->root_worker; |
|
|
|
|
GPR_ASSERT(new_root->initialized_cv); |
|
|
|
|
gpr_cv_signal(&new_root->cv); |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
case WRR_EMPTIED: |
|
|
|
|
if (pollset->active_pollable != worker->pollable_obj) { |
|
|
|
|
// pollable no longer being polled: flush events
|
|
|
|
|
pollable_process_events(exec_ctx, pollset, worker->pollable_obj, true); |
|
|
|
|
} |
|
|
|
|
break; |
|
|
|
|
case WRR_REMOVED: |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
gpr_mu_unlock(&worker->pollable_obj->mu); |
|
|
|
|
POLLABLE_UNREF(worker->pollable_obj, "pollset_worker"); |
|
|
|
|
gpr_mu_lock(&pollset->mu); |
|
|
|
|
if (worker_remove(&pollset->root_worker, worker, PWLINK_POLLSET) == |
|
|
|
|
WRR_EMPTIED) { |
|
|
|
|
pollset_maybe_finish_shutdown(exec_ctx, pollset); |
|
|
|
@ -957,12 +965,14 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
if (begin_worker(exec_ctx, pollset, WORKER_PTR, worker_hdl, deadline)) { |
|
|
|
|
gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); |
|
|
|
|
gpr_tls_set(&g_current_thread_worker, (intptr_t)WORKER_PTR); |
|
|
|
|
if (pollset->event_cursor == pollset->event_count) { |
|
|
|
|
if (WORKER_PTR->pollable_obj->event_cursor == WORKER_PTR->pollable_obj->event_count) { |
|
|
|
|
append_error(&error, pollset_epoll(exec_ctx, pollset, |
|
|
|
|
WORKER_PTR->pollable_obj, deadline), |
|
|
|
|
err_desc); |
|
|
|
|
} |
|
|
|
|
append_error(&error, pollset_process_events(exec_ctx, pollset, false), |
|
|
|
|
append_error(&error, |
|
|
|
|
pollable_process_events( |
|
|
|
|
exec_ctx, pollset, WORKER_PTR->pollable_obj, false), |
|
|
|
|
err_desc); |
|
|
|
|
grpc_exec_ctx_flush(exec_ctx); |
|
|
|
|
gpr_tls_set(&g_current_thread_pollset, 0); |
|
|
|
@ -973,6 +983,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, |
|
|
|
|
#ifdef GRPC_EPOLLEX_CREATE_WORKERS_ON_HEAP |
|
|
|
|
gpr_free(worker); |
|
|
|
|
#endif |
|
|
|
|
#undef WORKER_PTR |
|
|
|
|
return error; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|