|
|
|
@ -41,7 +41,14 @@ |
|
|
|
|
typedef bool (*bpstate_func)(grpc_exec_ctx *exec_ctx, |
|
|
|
|
grpc_buffer_pool *buffer_pool); |
|
|
|
|
|
|
|
|
|
typedef struct { |
|
|
|
|
grpc_buffer_user *head; |
|
|
|
|
grpc_buffer_user *tail; |
|
|
|
|
} grpc_buffer_user_list; |
|
|
|
|
|
|
|
|
|
struct grpc_buffer_pool { |
|
|
|
|
gpr_refcount refs; |
|
|
|
|
|
|
|
|
|
grpc_combiner *combiner; |
|
|
|
|
int64_t size; |
|
|
|
|
int64_t free_pool; |
|
|
|
@ -49,17 +56,57 @@ struct grpc_buffer_pool { |
|
|
|
|
bool step_scheduled; |
|
|
|
|
bool reclaiming; |
|
|
|
|
grpc_closure bpstep_closure; |
|
|
|
|
|
|
|
|
|
grpc_buffer_user_list lists[GRPC_BULIST_COUNT]; |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
/*******************************************************************************
|
|
|
|
|
* list management |
|
|
|
|
*/ |
|
|
|
|
|
|
|
|
|
void bulist_add(grpc_buffer_user *buffer_user, grpc_bulist list); |
|
|
|
|
bool bulist_empty(grpc_buffer_pool *buffer_pool, grpc_bulist list); |
|
|
|
|
grpc_buffer_user *bulist_head(grpc_buffer_pool *buffer_pool, grpc_bulist list); |
|
|
|
|
grpc_buffer_user *bulist_pop(grpc_buffer_pool *buffer_pool, grpc_bulist list); |
|
|
|
|
void bulist_remove(grpc_buffer_user *buffer_pool, grpc_bulist list); |
|
|
|
|
static void bulist_add_tail(grpc_buffer_user *buffer_user, grpc_bulist list) { |
|
|
|
|
grpc_buffer_pool *buffer_pool = buffer_user->buffer_pool; |
|
|
|
|
grpc_buffer_user_list *lst = &buffer_pool->lists[list]; |
|
|
|
|
if (lst->head == NULL) { |
|
|
|
|
lst->head = lst->tail = buffer_user; |
|
|
|
|
} else { |
|
|
|
|
lst->tail->next[list] = buffer_user; |
|
|
|
|
lst->tail = buffer_user; |
|
|
|
|
} |
|
|
|
|
buffer_user->next[list] = NULL; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void bulist_add_head(grpc_buffer_user *buffer_user, grpc_bulist list) { |
|
|
|
|
grpc_buffer_pool *buffer_pool = buffer_user->buffer_pool; |
|
|
|
|
grpc_buffer_user_list *lst = &buffer_pool->lists[list]; |
|
|
|
|
if (lst->head == NULL) { |
|
|
|
|
lst->head = lst->tail = buffer_user; |
|
|
|
|
buffer_user->next[list] = NULL; |
|
|
|
|
} else { |
|
|
|
|
buffer_user->next[list] = lst->head; |
|
|
|
|
lst->head = buffer_user; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static bool bulist_empty(grpc_buffer_pool *buffer_pool, grpc_bulist list) { |
|
|
|
|
return buffer_pool->lists[list].head == NULL; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static grpc_buffer_user *bulist_pop(grpc_buffer_pool *buffer_pool, |
|
|
|
|
grpc_bulist list) { |
|
|
|
|
grpc_buffer_user_list *lst = &buffer_pool->lists[list]; |
|
|
|
|
grpc_buffer_user *buffer_user = lst->head; |
|
|
|
|
if (buffer_user == NULL) { |
|
|
|
|
return NULL; |
|
|
|
|
} |
|
|
|
|
if (buffer_user == lst->tail) { |
|
|
|
|
lst->head = lst->tail = NULL; |
|
|
|
|
} else { |
|
|
|
|
lst->head = buffer_user->next[list]; |
|
|
|
|
} |
|
|
|
|
buffer_user->next[list] = NULL; |
|
|
|
|
return buffer_user; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/*******************************************************************************
|
|
|
|
|
* buffer pool state machine |
|
|
|
@ -93,7 +140,7 @@ static void bpstep_sched(grpc_exec_ctx *exec_ctx, |
|
|
|
|
static bool bpalloc(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool) { |
|
|
|
|
grpc_buffer_user *buffer_user; |
|
|
|
|
while ((buffer_user = |
|
|
|
|
bulist_head(buffer_pool, GRPC_BULIST_AWAITING_ALLOCATION))) { |
|
|
|
|
bulist_pop(buffer_pool, GRPC_BULIST_AWAITING_ALLOCATION))) { |
|
|
|
|
gpr_mu_lock(&buffer_user->mu); |
|
|
|
|
if (buffer_user->free_pool < 0 && |
|
|
|
|
-buffer_user->free_pool < buffer_pool->free_pool) { |
|
|
|
@ -103,9 +150,9 @@ static bool bpalloc(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool) { |
|
|
|
|
if (buffer_user->free_pool >= 0) { |
|
|
|
|
buffer_user->allocating = false; |
|
|
|
|
grpc_exec_ctx_enqueue_list(exec_ctx, &buffer_user->on_allocated, NULL); |
|
|
|
|
bulist_remove(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
|
|
|
|
gpr_mu_unlock(&buffer_user->mu); |
|
|
|
|
} else { |
|
|
|
|
bulist_add_head(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
|
|
|
|
gpr_mu_unlock(&buffer_user->mu); |
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
@ -155,7 +202,7 @@ static void bu_allocate(grpc_exec_ctx *exec_ctx, void *bu, grpc_error *error) { |
|
|
|
|
if (bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_AWAITING_ALLOCATION)) { |
|
|
|
|
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
|
|
|
|
} |
|
|
|
|
bulist_add(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
|
|
|
|
bulist_add_tail(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void bu_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
@ -166,7 +213,7 @@ static void bu_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
|
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_NON_EMPTY_FREE_POOL)) { |
|
|
|
|
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
|
|
|
|
} |
|
|
|
|
bulist_add(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
|
|
|
|
bulist_add_tail(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void bu_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
@ -178,7 +225,7 @@ static void bu_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
|
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_RECLAIMER_BENIGN)) { |
|
|
|
|
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
|
|
|
|
} |
|
|
|
|
bulist_add(buffer_user, GRPC_BULIST_RECLAIMER_BENIGN); |
|
|
|
|
bulist_add_tail(buffer_user, GRPC_BULIST_RECLAIMER_BENIGN); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static void bu_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
@ -192,7 +239,7 @@ static void bu_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
|
GRPC_BULIST_RECLAIMER_DESTRUCTIVE)) { |
|
|
|
|
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
|
|
|
|
} |
|
|
|
|
bulist_add(buffer_user, GRPC_BULIST_RECLAIMER_DESTRUCTIVE); |
|
|
|
|
bulist_add_tail(buffer_user, GRPC_BULIST_RECLAIMER_DESTRUCTIVE); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/*******************************************************************************
|
|
|
|
@ -201,6 +248,7 @@ static void bu_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
|
|
|
|
|
|
|
|
|
grpc_buffer_pool *grpc_buffer_pool_create(void) { |
|
|
|
|
grpc_buffer_pool *buffer_pool = gpr_malloc(sizeof(*buffer_pool)); |
|
|
|
|
gpr_ref_init(&buffer_pool->refs, 1); |
|
|
|
|
buffer_pool->combiner = grpc_combiner_create(NULL); |
|
|
|
|
buffer_pool->free_pool = INT64_MAX; |
|
|
|
|
buffer_pool->size = INT64_MAX; |
|
|
|
@ -208,13 +256,36 @@ grpc_buffer_pool *grpc_buffer_pool_create(void) { |
|
|
|
|
return buffer_pool; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void grpc_buffer_pool_internal_unref(grpc_exec_ctx *exec_ctx, |
|
|
|
|
grpc_buffer_pool *buffer_pool) { |
|
|
|
|
if (gpr_unref(&buffer_pool->refs)) { |
|
|
|
|
grpc_combiner_destroy(exec_ctx, buffer_pool->combiner); |
|
|
|
|
gpr_free(buffer_pool); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void grpc_buffer_pool_unref(grpc_buffer_pool *buffer_pool) { |
|
|
|
|
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
|
|
|
|
grpc_buffer_pool_internal_unref(&exec_ctx, buffer_pool); |
|
|
|
|
grpc_exec_ctx_finish(&exec_ctx); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_buffer_pool *grpc_buffer_pool_internal_ref(grpc_buffer_pool *buffer_pool) { |
|
|
|
|
gpr_ref(&buffer_pool->refs); |
|
|
|
|
return buffer_pool; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void grpc_buffer_pool_ref(grpc_buffer_pool *buffer_pool) { |
|
|
|
|
grpc_buffer_pool_internal_ref(buffer_pool); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/*******************************************************************************
|
|
|
|
|
* grpc_buffer_user api |
|
|
|
|
*/ |
|
|
|
|
|
|
|
|
|
void grpc_buffer_user_init(grpc_buffer_user *buffer_user, |
|
|
|
|
grpc_buffer_pool *buffer_pool) { |
|
|
|
|
buffer_user->buffer_pool = buffer_pool; |
|
|
|
|
buffer_user->buffer_pool = grpc_buffer_pool_internal_ref(buffer_pool); |
|
|
|
|
grpc_closure_init(&buffer_user->allocate_closure, &bu_allocate, buffer_user); |
|
|
|
|
grpc_closure_init(&buffer_user->add_to_free_pool_closure, |
|
|
|
|
&bu_add_to_free_pool, buffer_user); |
|
|
|
|