Craig Tiller
9 years ago
parent
654b242ce7
commit
20afa3d7c9
93 changed files with 2243 additions and 1871 deletions
@ -1,684 +0,0 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2016, Google Inc. |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or without |
||||
* modification, are permitted provided that the following conditions are |
||||
* met: |
||||
* |
||||
* * Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* * Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following disclaimer |
||||
* in the documentation and/or other materials provided with the |
||||
* distribution. |
||||
* * Neither the name of Google Inc. nor the names of its |
||||
* contributors may be used to endorse or promote products derived from |
||||
* this software without specific prior written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
* |
||||
*/ |
||||
|
||||
#include "src/core/lib/iomgr/buffer_pool.h" |
||||
|
||||
#include <string.h> |
||||
|
||||
#include <grpc/support/alloc.h> |
||||
#include <grpc/support/log.h> |
||||
#include <grpc/support/string_util.h> |
||||
#include <grpc/support/useful.h> |
||||
|
||||
#include "src/core/lib/iomgr/combiner.h" |
||||
|
||||
int grpc_buffer_pool_trace = 0; |
||||
|
||||
typedef bool (*bpstate_func)(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_pool *buffer_pool); |
||||
|
||||
typedef struct { |
||||
grpc_buffer_user *head; |
||||
grpc_buffer_user *tail; |
||||
} grpc_buffer_user_list; |
||||
|
||||
struct grpc_buffer_pool { |
||||
gpr_refcount refs; |
||||
|
||||
grpc_combiner *combiner; |
||||
int64_t size; |
||||
int64_t free_pool; |
||||
|
||||
bool step_scheduled; |
||||
bool reclaiming; |
||||
grpc_closure bpstep_closure; |
||||
grpc_closure bpreclaimation_done_closure; |
||||
|
||||
grpc_buffer_user *roots[GRPC_BULIST_COUNT]; |
||||
|
||||
char *name; |
||||
}; |
||||
|
||||
/*******************************************************************************
|
||||
* list management |
||||
*/ |
||||
|
||||
static void bulist_add_tail(grpc_buffer_user *buffer_user, grpc_bulist list) { |
||||
grpc_buffer_pool *buffer_pool = buffer_user->buffer_pool; |
||||
grpc_buffer_user **root = &buffer_pool->roots[list]; |
||||
if (*root == NULL) { |
||||
*root = buffer_user; |
||||
buffer_user->links[list].next = buffer_user->links[list].prev = buffer_user; |
||||
} else { |
||||
buffer_user->links[list].next = *root; |
||||
buffer_user->links[list].prev = (*root)->links[list].prev; |
||||
buffer_user->links[list].next->links[list].prev = |
||||
buffer_user->links[list].prev->links[list].next = buffer_user; |
||||
} |
||||
} |
||||
|
||||
static void bulist_add_head(grpc_buffer_user *buffer_user, grpc_bulist list) { |
||||
grpc_buffer_pool *buffer_pool = buffer_user->buffer_pool; |
||||
grpc_buffer_user **root = &buffer_pool->roots[list]; |
||||
if (*root == NULL) { |
||||
*root = buffer_user; |
||||
buffer_user->links[list].next = buffer_user->links[list].prev = buffer_user; |
||||
} else { |
||||
buffer_user->links[list].next = (*root)->links[list].next; |
||||
buffer_user->links[list].prev = *root; |
||||
buffer_user->links[list].next->links[list].prev = |
||||
buffer_user->links[list].prev->links[list].next = buffer_user; |
||||
*root = buffer_user; |
||||
} |
||||
} |
||||
|
||||
static bool bulist_empty(grpc_buffer_pool *buffer_pool, grpc_bulist list) { |
||||
return buffer_pool->roots[list] == NULL; |
||||
} |
||||
|
||||
static grpc_buffer_user *bulist_pop(grpc_buffer_pool *buffer_pool, |
||||
grpc_bulist list) { |
||||
grpc_buffer_user **root = &buffer_pool->roots[list]; |
||||
grpc_buffer_user *buffer_user = *root; |
||||
if (buffer_user == NULL) { |
||||
return NULL; |
||||
} |
||||
if (buffer_user->links[list].next == buffer_user) { |
||||
*root = NULL; |
||||
} else { |
||||
buffer_user->links[list].next->links[list].prev = |
||||
buffer_user->links[list].prev; |
||||
buffer_user->links[list].prev->links[list].next = |
||||
buffer_user->links[list].next; |
||||
*root = buffer_user->links[list].next; |
||||
} |
||||
buffer_user->links[list].next = buffer_user->links[list].prev = NULL; |
||||
return buffer_user; |
||||
} |
||||
|
||||
static void bulist_remove(grpc_buffer_user *buffer_user, grpc_bulist list) { |
||||
if (buffer_user->links[list].next == NULL) return; |
||||
grpc_buffer_pool *buffer_pool = buffer_user->buffer_pool; |
||||
if (buffer_pool->roots[list] == buffer_user) { |
||||
buffer_pool->roots[list] = buffer_user->links[list].next; |
||||
if (buffer_pool->roots[list] == buffer_user) { |
||||
buffer_pool->roots[list] = NULL; |
||||
} |
||||
} |
||||
buffer_user->links[list].next->links[list].prev = |
||||
buffer_user->links[list].prev; |
||||
buffer_user->links[list].prev->links[list].next = |
||||
buffer_user->links[list].next; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* buffer pool state machine |
||||
*/ |
||||
|
||||
static bool bpalloc(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool); |
||||
static bool bpscavenge(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool); |
||||
static bool bpreclaim(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool, |
||||
bool destructive); |
||||
|
||||
static void bpstep(grpc_exec_ctx *exec_ctx, void *bp, grpc_error *error) { |
||||
grpc_buffer_pool *buffer_pool = bp; |
||||
buffer_pool->step_scheduled = false; |
||||
do { |
||||
if (bpalloc(exec_ctx, buffer_pool)) goto done; |
||||
} while (bpscavenge(exec_ctx, buffer_pool)); |
||||
bpreclaim(exec_ctx, buffer_pool, false) || |
||||
bpreclaim(exec_ctx, buffer_pool, true); |
||||
done: |
||||
grpc_buffer_pool_internal_unref(exec_ctx, buffer_pool); |
||||
} |
||||
|
||||
static void bpstep_sched(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_pool *buffer_pool) { |
||||
if (buffer_pool->step_scheduled) return; |
||||
buffer_pool->step_scheduled = true; |
||||
grpc_buffer_pool_internal_ref(buffer_pool); |
||||
grpc_combiner_execute_finally(exec_ctx, buffer_pool->combiner, |
||||
&buffer_pool->bpstep_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
|
||||
/* returns true if all allocations are completed */ |
||||
static bool bpalloc(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool) { |
||||
grpc_buffer_user *buffer_user; |
||||
while ((buffer_user = |
||||
bulist_pop(buffer_pool, GRPC_BULIST_AWAITING_ALLOCATION))) { |
||||
gpr_mu_lock(&buffer_user->mu); |
||||
if (buffer_user->free_pool < 0 && |
||||
-buffer_user->free_pool <= buffer_pool->free_pool) { |
||||
int64_t amt = -buffer_user->free_pool; |
||||
buffer_user->free_pool = 0; |
||||
buffer_pool->free_pool -= amt; |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: grant alloc %" PRId64 |
||||
" bytes; bp_free_pool -> %" PRId64, |
||||
buffer_pool->name, buffer_user->name, amt, |
||||
buffer_pool->free_pool); |
||||
} |
||||
} else if (grpc_buffer_pool_trace && buffer_user->free_pool >= 0) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: discard already satisfied alloc request", |
||||
buffer_pool->name, buffer_user->name); |
||||
} |
||||
if (buffer_user->free_pool >= 0) { |
||||
buffer_user->allocating = false; |
||||
grpc_exec_ctx_enqueue_list(exec_ctx, &buffer_user->on_allocated, NULL); |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
} else { |
||||
bulist_add_head(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
/* returns true if any memory could be reclaimed from buffers */ |
||||
static bool bpscavenge(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool) { |
||||
grpc_buffer_user *buffer_user; |
||||
while ((buffer_user = |
||||
bulist_pop(buffer_pool, GRPC_BULIST_NON_EMPTY_FREE_POOL))) { |
||||
gpr_mu_lock(&buffer_user->mu); |
||||
if (buffer_user->free_pool > 0) { |
||||
int64_t amt = buffer_user->free_pool; |
||||
buffer_user->free_pool = 0; |
||||
buffer_pool->free_pool += amt; |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: scavenge %" PRId64 |
||||
" bytes; bp_free_pool -> %" PRId64, |
||||
buffer_pool->name, buffer_user->name, amt, |
||||
buffer_pool->free_pool); |
||||
} |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
return true; |
||||
} else { |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/* returns true if reclaimation is proceeding */ |
||||
static bool bpreclaim(grpc_exec_ctx *exec_ctx, grpc_buffer_pool *buffer_pool, |
||||
bool destructive) { |
||||
if (buffer_pool->reclaiming) return true; |
||||
grpc_bulist list = destructive ? GRPC_BULIST_RECLAIMER_DESTRUCTIVE |
||||
: GRPC_BULIST_RECLAIMER_BENIGN; |
||||
grpc_buffer_user *buffer_user = bulist_pop(buffer_pool, list); |
||||
if (buffer_user == NULL) return false; |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: initiate %s reclaimation", buffer_pool->name, |
||||
buffer_user->name, destructive ? "destructive" : "benign"); |
||||
} |
||||
buffer_pool->reclaiming = true; |
||||
grpc_buffer_pool_internal_ref(buffer_pool); |
||||
grpc_closure *c = buffer_user->reclaimers[destructive]; |
||||
buffer_user->reclaimers[destructive] = NULL; |
||||
grpc_closure_run(exec_ctx, c, GRPC_ERROR_NONE); |
||||
return true; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* bu_slice: a slice implementation that is backed by a grpc_buffer_user |
||||
*/ |
||||
|
||||
typedef struct { |
||||
gpr_slice_refcount base; |
||||
gpr_refcount refs; |
||||
grpc_buffer_user *buffer_user; |
||||
size_t size; |
||||
} bu_slice_refcount; |
||||
|
||||
static void bu_slice_ref(void *p) { |
||||
bu_slice_refcount *rc = p; |
||||
gpr_ref(&rc->refs); |
||||
} |
||||
|
||||
static void bu_slice_unref(void *p) { |
||||
bu_slice_refcount *rc = p; |
||||
if (gpr_unref(&rc->refs)) { |
||||
/* TODO(ctiller): this is dangerous, but I think safe for now:
|
||||
we have no guarantee here that we're at a safe point for creating an |
||||
execution context, but we have no way of writing this code otherwise. |
||||
In the future: consider lifting gpr_slice to grpc, and offering an |
||||
internal_{ref,unref} pair that is execution context aware. Alternatively, |
||||
make exec_ctx be thread local and 'do the right thing' (whatever that is) |
||||
if NULL */ |
||||
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
||||
grpc_buffer_user_free(&exec_ctx, rc->buffer_user, rc->size); |
||||
grpc_exec_ctx_finish(&exec_ctx); |
||||
gpr_free(rc); |
||||
} |
||||
} |
||||
|
||||
static gpr_slice bu_slice_create(grpc_buffer_user *buffer_user, size_t size) { |
||||
bu_slice_refcount *rc = gpr_malloc(sizeof(bu_slice_refcount) + size); |
||||
rc->base.ref = bu_slice_ref; |
||||
rc->base.unref = bu_slice_unref; |
||||
gpr_ref_init(&rc->refs, 1); |
||||
rc->buffer_user = buffer_user; |
||||
rc->size = size; |
||||
gpr_slice slice; |
||||
slice.refcount = &rc->base; |
||||
slice.data.refcounted.bytes = (uint8_t *)(rc + 1); |
||||
slice.data.refcounted.length = size; |
||||
return slice; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_buffer_pool internal implementation |
||||
*/ |
||||
|
||||
static void bu_allocate(grpc_exec_ctx *exec_ctx, void *bu, grpc_error *error) { |
||||
grpc_buffer_user *buffer_user = bu; |
||||
if (bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_AWAITING_ALLOCATION)) { |
||||
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
||||
} |
||||
bulist_add_tail(buffer_user, GRPC_BULIST_AWAITING_ALLOCATION); |
||||
} |
||||
|
||||
static void bu_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *bu, |
||||
grpc_error *error) { |
||||
grpc_buffer_user *buffer_user = bu; |
||||
if (!bulist_empty(buffer_user->buffer_pool, |
||||
GRPC_BULIST_AWAITING_ALLOCATION) && |
||||
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_NON_EMPTY_FREE_POOL)) { |
||||
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
||||
} |
||||
bulist_add_tail(buffer_user, GRPC_BULIST_NON_EMPTY_FREE_POOL); |
||||
} |
||||
|
||||
static void bu_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
||||
grpc_error *error) { |
||||
grpc_buffer_user *buffer_user = bu; |
||||
if (!bulist_empty(buffer_user->buffer_pool, |
||||
GRPC_BULIST_AWAITING_ALLOCATION) && |
||||
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_NON_EMPTY_FREE_POOL) && |
||||
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_RECLAIMER_BENIGN)) { |
||||
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
||||
} |
||||
bulist_add_tail(buffer_user, GRPC_BULIST_RECLAIMER_BENIGN); |
||||
} |
||||
|
||||
static void bu_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
||||
grpc_error *error) { |
||||
grpc_buffer_user *buffer_user = bu; |
||||
if (!bulist_empty(buffer_user->buffer_pool, |
||||
GRPC_BULIST_AWAITING_ALLOCATION) && |
||||
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_NON_EMPTY_FREE_POOL) && |
||||
bulist_empty(buffer_user->buffer_pool, GRPC_BULIST_RECLAIMER_BENIGN) && |
||||
bulist_empty(buffer_user->buffer_pool, |
||||
GRPC_BULIST_RECLAIMER_DESTRUCTIVE)) { |
||||
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
||||
} |
||||
bulist_add_tail(buffer_user, GRPC_BULIST_RECLAIMER_DESTRUCTIVE); |
||||
} |
||||
|
||||
static void bu_destroy(grpc_exec_ctx *exec_ctx, void *bu, grpc_error *error) { |
||||
grpc_buffer_user *buffer_user = bu; |
||||
GPR_ASSERT(buffer_user->allocated == 0); |
||||
for (int i = 0; i < GRPC_BULIST_COUNT; i++) { |
||||
bulist_remove(buffer_user, (grpc_bulist)i); |
||||
} |
||||
grpc_exec_ctx_sched(exec_ctx, buffer_user->reclaimers[0], |
||||
GRPC_ERROR_CANCELLED, NULL); |
||||
grpc_exec_ctx_sched(exec_ctx, buffer_user->reclaimers[1], |
||||
GRPC_ERROR_CANCELLED, NULL); |
||||
grpc_exec_ctx_sched(exec_ctx, (grpc_closure *)gpr_atm_no_barrier_load( |
||||
&buffer_user->on_done_destroy_closure), |
||||
GRPC_ERROR_NONE, NULL); |
||||
if (buffer_user->free_pool != 0) { |
||||
buffer_user->buffer_pool->free_pool += buffer_user->free_pool; |
||||
bpstep_sched(exec_ctx, buffer_user->buffer_pool); |
||||
} |
||||
} |
||||
|
||||
static void bu_allocated_slices(grpc_exec_ctx *exec_ctx, void *ts, |
||||
grpc_error *error) { |
||||
grpc_buffer_user_slice_allocator *slice_allocator = ts; |
||||
if (error == GRPC_ERROR_NONE) { |
||||
for (size_t i = 0; i < slice_allocator->count; i++) { |
||||
gpr_slice_buffer_add_indexed(slice_allocator->dest, |
||||
bu_slice_create(slice_allocator->buffer_user, |
||||
slice_allocator->length)); |
||||
} |
||||
} |
||||
grpc_closure_run(exec_ctx, &slice_allocator->on_done, GRPC_ERROR_REF(error)); |
||||
} |
||||
|
||||
typedef struct { |
||||
int64_t size; |
||||
grpc_buffer_pool *buffer_pool; |
||||
grpc_closure closure; |
||||
} bp_resize_args; |
||||
|
||||
static void bp_resize(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) { |
||||
bp_resize_args *a = args; |
||||
int64_t delta = a->size - a->buffer_pool->size; |
||||
a->buffer_pool->size += delta; |
||||
a->buffer_pool->free_pool += delta; |
||||
if (delta < 0 && a->buffer_pool->free_pool < 0) { |
||||
bpstep_sched(exec_ctx, a->buffer_pool); |
||||
} else if (delta > 0 && |
||||
!bulist_empty(a->buffer_pool, GRPC_BULIST_AWAITING_ALLOCATION)) { |
||||
bpstep_sched(exec_ctx, a->buffer_pool); |
||||
} |
||||
grpc_buffer_pool_internal_unref(exec_ctx, a->buffer_pool); |
||||
gpr_free(a); |
||||
} |
||||
|
||||
static void bp_reclaimation_done(grpc_exec_ctx *exec_ctx, void *bp, |
||||
grpc_error *error) { |
||||
grpc_buffer_pool *buffer_pool = bp; |
||||
buffer_pool->reclaiming = false; |
||||
bpstep_sched(exec_ctx, buffer_pool); |
||||
grpc_buffer_pool_internal_unref(exec_ctx, buffer_pool); |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_buffer_pool api |
||||
*/ |
||||
|
||||
grpc_buffer_pool *grpc_buffer_pool_create(const char *name) { |
||||
grpc_buffer_pool *buffer_pool = gpr_malloc(sizeof(*buffer_pool)); |
||||
gpr_ref_init(&buffer_pool->refs, 1); |
||||
buffer_pool->combiner = grpc_combiner_create(NULL); |
||||
buffer_pool->free_pool = INT64_MAX; |
||||
buffer_pool->size = INT64_MAX; |
||||
buffer_pool->step_scheduled = false; |
||||
buffer_pool->reclaiming = false; |
||||
if (name != NULL) { |
||||
buffer_pool->name = gpr_strdup(name); |
||||
} else { |
||||
gpr_asprintf(&buffer_pool->name, "anonymous_pool_%" PRIxPTR, |
||||
(intptr_t)buffer_pool); |
||||
} |
||||
grpc_closure_init(&buffer_pool->bpstep_closure, bpstep, buffer_pool); |
||||
grpc_closure_init(&buffer_pool->bpreclaimation_done_closure, |
||||
bp_reclaimation_done, buffer_pool); |
||||
for (int i = 0; i < GRPC_BULIST_COUNT; i++) { |
||||
buffer_pool->roots[i] = NULL; |
||||
} |
||||
return buffer_pool; |
||||
} |
||||
|
||||
void grpc_buffer_pool_internal_unref(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_pool *buffer_pool) { |
||||
if (gpr_unref(&buffer_pool->refs)) { |
||||
grpc_combiner_destroy(exec_ctx, buffer_pool->combiner); |
||||
gpr_free(buffer_pool->name); |
||||
gpr_free(buffer_pool); |
||||
} |
||||
} |
||||
|
||||
void grpc_buffer_pool_unref(grpc_buffer_pool *buffer_pool) { |
||||
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
||||
grpc_buffer_pool_internal_unref(&exec_ctx, buffer_pool); |
||||
grpc_exec_ctx_finish(&exec_ctx); |
||||
} |
||||
|
||||
grpc_buffer_pool *grpc_buffer_pool_internal_ref(grpc_buffer_pool *buffer_pool) { |
||||
gpr_ref(&buffer_pool->refs); |
||||
return buffer_pool; |
||||
} |
||||
|
||||
void grpc_buffer_pool_ref(grpc_buffer_pool *buffer_pool) { |
||||
grpc_buffer_pool_internal_ref(buffer_pool); |
||||
} |
||||
|
||||
void grpc_buffer_pool_resize(grpc_buffer_pool *buffer_pool, size_t size) { |
||||
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
||||
bp_resize_args *a = gpr_malloc(sizeof(*a)); |
||||
a->buffer_pool = grpc_buffer_pool_internal_ref(buffer_pool); |
||||
a->size = (int64_t)size; |
||||
grpc_closure_init(&a->closure, bp_resize, a); |
||||
grpc_combiner_execute(&exec_ctx, buffer_pool->combiner, &a->closure, |
||||
GRPC_ERROR_NONE, false); |
||||
grpc_exec_ctx_finish(&exec_ctx); |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_buffer_user channel args api |
||||
*/ |
||||
|
||||
grpc_buffer_pool *grpc_buffer_pool_from_channel_args( |
||||
const grpc_channel_args *channel_args) { |
||||
for (size_t i = 0; i < channel_args->num_args; i++) { |
||||
if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_BUFFER_POOL)) { |
||||
if (channel_args->args[i].type == GRPC_ARG_POINTER) { |
||||
return grpc_buffer_pool_internal_ref( |
||||
channel_args->args[i].value.pointer.p); |
||||
} else { |
||||
gpr_log(GPR_DEBUG, GRPC_ARG_BUFFER_POOL " should be a pointer"); |
||||
} |
||||
} |
||||
} |
||||
return grpc_buffer_pool_create(NULL); |
||||
} |
||||
|
||||
static void *bp_copy(void *bp) { |
||||
grpc_buffer_pool_ref(bp); |
||||
return bp; |
||||
} |
||||
|
||||
static void bp_destroy(void *bp) { grpc_buffer_pool_unref(bp); } |
||||
|
||||
static int bp_cmp(void *a, void *b) { return GPR_ICMP(a, b); } |
||||
|
||||
const grpc_arg_pointer_vtable *grpc_buffer_pool_arg_vtable(void) { |
||||
static const grpc_arg_pointer_vtable vtable = {bp_copy, bp_destroy, bp_cmp}; |
||||
return &vtable; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_buffer_user api |
||||
*/ |
||||
|
||||
void grpc_buffer_user_init(grpc_buffer_user *buffer_user, |
||||
grpc_buffer_pool *buffer_pool, const char *name) { |
||||
buffer_user->buffer_pool = grpc_buffer_pool_internal_ref(buffer_pool); |
||||
grpc_closure_init(&buffer_user->allocate_closure, &bu_allocate, buffer_user); |
||||
grpc_closure_init(&buffer_user->add_to_free_pool_closure, |
||||
&bu_add_to_free_pool, buffer_user); |
||||
grpc_closure_init(&buffer_user->post_reclaimer_closure[0], |
||||
&bu_post_benign_reclaimer, buffer_user); |
||||
grpc_closure_init(&buffer_user->post_reclaimer_closure[1], |
||||
&bu_post_destructive_reclaimer, buffer_user); |
||||
grpc_closure_init(&buffer_user->destroy_closure, &bu_destroy, buffer_user); |
||||
gpr_mu_init(&buffer_user->mu); |
||||
buffer_user->allocated = 0; |
||||
buffer_user->free_pool = 0; |
||||
grpc_closure_list_init(&buffer_user->on_allocated); |
||||
buffer_user->allocating = false; |
||||
buffer_user->added_to_free_pool = false; |
||||
gpr_atm_no_barrier_store(&buffer_user->on_done_destroy_closure, 0); |
||||
buffer_user->reclaimers[0] = NULL; |
||||
buffer_user->reclaimers[1] = NULL; |
||||
for (int i = 0; i < GRPC_BULIST_COUNT; i++) { |
||||
buffer_user->links[i].next = buffer_user->links[i].prev = NULL; |
||||
} |
||||
#ifndef NDEBUG |
||||
buffer_user->asan_canary = gpr_malloc(1); |
||||
#endif |
||||
if (name != NULL) { |
||||
buffer_user->name = gpr_strdup(name); |
||||
} else { |
||||
gpr_asprintf(&buffer_user->name, "anonymous_buffer_user_%" PRIxPTR, |
||||
(intptr_t)buffer_user); |
||||
} |
||||
} |
||||
|
||||
void grpc_buffer_user_shutdown(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_user *buffer_user, |
||||
grpc_closure *on_done) { |
||||
gpr_mu_lock(&buffer_user->mu); |
||||
GPR_ASSERT(gpr_atm_no_barrier_load(&buffer_user->on_done_destroy_closure) == |
||||
0); |
||||
gpr_atm_no_barrier_store(&buffer_user->on_done_destroy_closure, |
||||
(gpr_atm)on_done); |
||||
if (buffer_user->allocated == 0) { |
||||
grpc_combiner_execute(exec_ctx, buffer_user->buffer_pool->combiner, |
||||
&buffer_user->destroy_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
} |
||||
|
||||
void grpc_buffer_user_destroy(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_user *buffer_user) { |
||||
#ifndef NDEBUG |
||||
gpr_free(buffer_user->asan_canary); |
||||
#endif |
||||
grpc_buffer_pool_internal_unref(exec_ctx, buffer_user->buffer_pool); |
||||
gpr_mu_destroy(&buffer_user->mu); |
||||
gpr_free(buffer_user->name); |
||||
} |
||||
|
||||
void grpc_buffer_user_alloc(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_user *buffer_user, size_t size, |
||||
grpc_closure *optional_on_done) { |
||||
gpr_mu_lock(&buffer_user->mu); |
||||
grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( |
||||
&buffer_user->on_done_destroy_closure); |
||||
if (on_done_destroy != NULL) { |
||||
/* already shutdown */ |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: alloc %" PRIdPTR " after shutdown", |
||||
buffer_user->buffer_pool->name, buffer_user->name, size); |
||||
} |
||||
grpc_exec_ctx_sched( |
||||
exec_ctx, optional_on_done, |
||||
GRPC_ERROR_CREATE("Buffer pool user is already shutdown"), NULL); |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
return; |
||||
} |
||||
buffer_user->allocated += (int64_t)size; |
||||
buffer_user->free_pool -= (int64_t)size; |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: alloc %" PRIdPTR "; allocated -> %" PRId64 |
||||
", free_pool -> %" PRId64, |
||||
buffer_user->buffer_pool->name, buffer_user->name, size, |
||||
buffer_user->allocated, buffer_user->free_pool); |
||||
} |
||||
if (buffer_user->free_pool < 0) { |
||||
grpc_closure_list_append(&buffer_user->on_allocated, optional_on_done, |
||||
GRPC_ERROR_NONE); |
||||
if (!buffer_user->allocating) { |
||||
buffer_user->allocating = true; |
||||
grpc_combiner_execute(exec_ctx, buffer_user->buffer_pool->combiner, |
||||
&buffer_user->allocate_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
} else { |
||||
grpc_exec_ctx_sched(exec_ctx, optional_on_done, GRPC_ERROR_NONE, NULL); |
||||
} |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
} |
||||
|
||||
void grpc_buffer_user_free(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_user *buffer_user, size_t size) { |
||||
gpr_mu_lock(&buffer_user->mu); |
||||
GPR_ASSERT(buffer_user->allocated >= (int64_t)size); |
||||
bool was_zero_or_negative = buffer_user->free_pool <= 0; |
||||
buffer_user->free_pool += (int64_t)size; |
||||
buffer_user->allocated -= (int64_t)size; |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: free %" PRIdPTR "; allocated -> %" PRId64 |
||||
", free_pool -> %" PRId64, |
||||
buffer_user->buffer_pool->name, buffer_user->name, size, |
||||
buffer_user->allocated, buffer_user->free_pool); |
||||
} |
||||
bool is_bigger_than_zero = buffer_user->free_pool > 0; |
||||
if (is_bigger_than_zero && was_zero_or_negative && |
||||
!buffer_user->added_to_free_pool) { |
||||
buffer_user->added_to_free_pool = true; |
||||
grpc_combiner_execute(exec_ctx, buffer_user->buffer_pool->combiner, |
||||
&buffer_user->add_to_free_pool_closure, |
||||
GRPC_ERROR_NONE, false); |
||||
} |
||||
grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( |
||||
&buffer_user->on_done_destroy_closure); |
||||
if (on_done_destroy != NULL && buffer_user->allocated == 0) { |
||||
grpc_combiner_execute(exec_ctx, buffer_user->buffer_pool->combiner, |
||||
&buffer_user->destroy_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
gpr_mu_unlock(&buffer_user->mu); |
||||
} |
||||
|
||||
void grpc_buffer_user_post_reclaimer(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_user *buffer_user, |
||||
bool destructive, grpc_closure *closure) { |
||||
if (gpr_atm_acq_load(&buffer_user->on_done_destroy_closure) == 0) { |
||||
GPR_ASSERT(buffer_user->reclaimers[destructive] == NULL); |
||||
buffer_user->reclaimers[destructive] = closure; |
||||
grpc_combiner_execute(exec_ctx, buffer_user->buffer_pool->combiner, |
||||
&buffer_user->post_reclaimer_closure[destructive], |
||||
GRPC_ERROR_NONE, false); |
||||
} else { |
||||
grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL); |
||||
} |
||||
} |
||||
|
||||
void grpc_buffer_user_finish_reclaimation(grpc_exec_ctx *exec_ctx, |
||||
grpc_buffer_user *buffer_user) { |
||||
if (grpc_buffer_pool_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: reclaimation complete", |
||||
buffer_user->buffer_pool->name, buffer_user->name); |
||||
} |
||||
grpc_combiner_execute(exec_ctx, buffer_user->buffer_pool->combiner, |
||||
&buffer_user->buffer_pool->bpreclaimation_done_closure, |
||||
GRPC_ERROR_NONE, false); |
||||
} |
||||
|
||||
void grpc_buffer_user_slice_allocator_init( |
||||
grpc_buffer_user_slice_allocator *slice_allocator, |
||||
grpc_buffer_user *buffer_user, grpc_iomgr_cb_func cb, void *p) { |
||||
grpc_closure_init(&slice_allocator->on_allocated, bu_allocated_slices, |
||||
slice_allocator); |
||||
grpc_closure_init(&slice_allocator->on_done, cb, p); |
||||
slice_allocator->buffer_user = buffer_user; |
||||
} |
||||
|
||||
void grpc_buffer_user_alloc_slices( |
||||
grpc_exec_ctx *exec_ctx, grpc_buffer_user_slice_allocator *slice_allocator, |
||||
size_t length, size_t count, gpr_slice_buffer *dest) { |
||||
slice_allocator->length = length; |
||||
slice_allocator->count = count; |
||||
slice_allocator->dest = dest; |
||||
grpc_buffer_user_alloc(exec_ctx, slice_allocator->buffer_user, count * length, |
||||
&slice_allocator->on_allocated); |
||||
} |
||||
@ -0,0 +1,684 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2016, Google Inc. |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or without |
||||
* modification, are permitted provided that the following conditions are |
||||
* met: |
||||
* |
||||
* * Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* * Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following disclaimer |
||||
* in the documentation and/or other materials provided with the |
||||
* distribution. |
||||
* * Neither the name of Google Inc. nor the names of its |
||||
* contributors may be used to endorse or promote products derived from |
||||
* this software without specific prior written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
* |
||||
*/ |
||||
|
||||
#include "src/core/lib/iomgr/resource_quota.h" |
||||
|
||||
#include <string.h> |
||||
|
||||
#include <grpc/support/alloc.h> |
||||
#include <grpc/support/log.h> |
||||
#include <grpc/support/string_util.h> |
||||
#include <grpc/support/useful.h> |
||||
|
||||
#include "src/core/lib/iomgr/combiner.h" |
||||
|
||||
int grpc_resource_quota_trace = 0; |
||||
|
||||
typedef bool (*bpstate_func)(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_quota *resource_quota); |
||||
|
||||
typedef struct { |
||||
grpc_resource_user *head; |
||||
grpc_resource_user *tail; |
||||
} grpc_resource_user_list; |
||||
|
||||
struct grpc_resource_quota { |
||||
gpr_refcount refs; |
||||
|
||||
grpc_combiner *combiner; |
||||
int64_t size; |
||||
int64_t free_pool; |
||||
|
||||
bool step_scheduled; |
||||
bool reclaiming; |
||||
grpc_closure bpstep_closure; |
||||
grpc_closure bpreclaimation_done_closure; |
||||
|
||||
grpc_resource_user *roots[GRPC_BULIST_COUNT]; |
||||
|
||||
char *name; |
||||
}; |
||||
|
||||
/*******************************************************************************
|
||||
* list management |
||||
*/ |
||||
|
||||
static void bulist_add_tail(grpc_resource_user *resource_user, grpc_bulist list) { |
||||
grpc_resource_quota *resource_quota = resource_user->resource_quota; |
||||
grpc_resource_user **root = &resource_quota->roots[list]; |
||||
if (*root == NULL) { |
||||
*root = resource_user; |
||||
resource_user->links[list].next = resource_user->links[list].prev = resource_user; |
||||
} else { |
||||
resource_user->links[list].next = *root; |
||||
resource_user->links[list].prev = (*root)->links[list].prev; |
||||
resource_user->links[list].next->links[list].prev = |
||||
resource_user->links[list].prev->links[list].next = resource_user; |
||||
} |
||||
} |
||||
|
||||
static void bulist_add_head(grpc_resource_user *resource_user, grpc_bulist list) { |
||||
grpc_resource_quota *resource_quota = resource_user->resource_quota; |
||||
grpc_resource_user **root = &resource_quota->roots[list]; |
||||
if (*root == NULL) { |
||||
*root = resource_user; |
||||
resource_user->links[list].next = resource_user->links[list].prev = resource_user; |
||||
} else { |
||||
resource_user->links[list].next = (*root)->links[list].next; |
||||
resource_user->links[list].prev = *root; |
||||
resource_user->links[list].next->links[list].prev = |
||||
resource_user->links[list].prev->links[list].next = resource_user; |
||||
*root = resource_user; |
||||
} |
||||
} |
||||
|
||||
static bool bulist_empty(grpc_resource_quota *resource_quota, grpc_bulist list) { |
||||
return resource_quota->roots[list] == NULL; |
||||
} |
||||
|
||||
static grpc_resource_user *bulist_pop(grpc_resource_quota *resource_quota, |
||||
grpc_bulist list) { |
||||
grpc_resource_user **root = &resource_quota->roots[list]; |
||||
grpc_resource_user *resource_user = *root; |
||||
if (resource_user == NULL) { |
||||
return NULL; |
||||
} |
||||
if (resource_user->links[list].next == resource_user) { |
||||
*root = NULL; |
||||
} else { |
||||
resource_user->links[list].next->links[list].prev = |
||||
resource_user->links[list].prev; |
||||
resource_user->links[list].prev->links[list].next = |
||||
resource_user->links[list].next; |
||||
*root = resource_user->links[list].next; |
||||
} |
||||
resource_user->links[list].next = resource_user->links[list].prev = NULL; |
||||
return resource_user; |
||||
} |
||||
|
||||
static void bulist_remove(grpc_resource_user *resource_user, grpc_bulist list) { |
||||
if (resource_user->links[list].next == NULL) return; |
||||
grpc_resource_quota *resource_quota = resource_user->resource_quota; |
||||
if (resource_quota->roots[list] == resource_user) { |
||||
resource_quota->roots[list] = resource_user->links[list].next; |
||||
if (resource_quota->roots[list] == resource_user) { |
||||
resource_quota->roots[list] = NULL; |
||||
} |
||||
} |
||||
resource_user->links[list].next->links[list].prev = |
||||
resource_user->links[list].prev; |
||||
resource_user->links[list].prev->links[list].next = |
||||
resource_user->links[list].next; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* buffer pool state machine |
||||
*/ |
||||
|
||||
static bool bpalloc(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota); |
||||
static bool bpscavenge(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota); |
||||
static bool bpreclaim(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota, |
||||
bool destructive); |
||||
|
||||
static void bpstep(grpc_exec_ctx *exec_ctx, void *bp, grpc_error *error) { |
||||
grpc_resource_quota *resource_quota = bp; |
||||
resource_quota->step_scheduled = false; |
||||
do { |
||||
if (bpalloc(exec_ctx, resource_quota)) goto done; |
||||
} while (bpscavenge(exec_ctx, resource_quota)); |
||||
bpreclaim(exec_ctx, resource_quota, false) || |
||||
bpreclaim(exec_ctx, resource_quota, true); |
||||
done: |
||||
grpc_resource_quota_internal_unref(exec_ctx, resource_quota); |
||||
} |
||||
|
||||
static void bpstep_sched(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_quota *resource_quota) { |
||||
if (resource_quota->step_scheduled) return; |
||||
resource_quota->step_scheduled = true; |
||||
grpc_resource_quota_internal_ref(resource_quota); |
||||
grpc_combiner_execute_finally(exec_ctx, resource_quota->combiner, |
||||
&resource_quota->bpstep_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
|
||||
/* returns true if all allocations are completed */ |
||||
static bool bpalloc(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { |
||||
grpc_resource_user *resource_user; |
||||
while ((resource_user = |
||||
bulist_pop(resource_quota, GRPC_BULIST_AWAITING_ALLOCATION))) { |
||||
gpr_mu_lock(&resource_user->mu); |
||||
if (resource_user->free_pool < 0 && |
||||
-resource_user->free_pool <= resource_quota->free_pool) { |
||||
int64_t amt = -resource_user->free_pool; |
||||
resource_user->free_pool = 0; |
||||
resource_quota->free_pool -= amt; |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: grant alloc %" PRId64 |
||||
" bytes; bp_free_pool -> %" PRId64, |
||||
resource_quota->name, resource_user->name, amt, |
||||
resource_quota->free_pool); |
||||
} |
||||
} else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: discard already satisfied alloc request", |
||||
resource_quota->name, resource_user->name); |
||||
} |
||||
if (resource_user->free_pool >= 0) { |
||||
resource_user->allocating = false; |
||||
grpc_exec_ctx_enqueue_list(exec_ctx, &resource_user->on_allocated, NULL); |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
} else { |
||||
bulist_add_head(resource_user, GRPC_BULIST_AWAITING_ALLOCATION); |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
return false; |
||||
} |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
/* returns true if any memory could be reclaimed from buffers */ |
||||
static bool bpscavenge(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) { |
||||
grpc_resource_user *resource_user; |
||||
while ((resource_user = |
||||
bulist_pop(resource_quota, GRPC_BULIST_NON_EMPTY_FREE_POOL))) { |
||||
gpr_mu_lock(&resource_user->mu); |
||||
if (resource_user->free_pool > 0) { |
||||
int64_t amt = resource_user->free_pool; |
||||
resource_user->free_pool = 0; |
||||
resource_quota->free_pool += amt; |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: scavenge %" PRId64 |
||||
" bytes; bp_free_pool -> %" PRId64, |
||||
resource_quota->name, resource_user->name, amt, |
||||
resource_quota->free_pool); |
||||
} |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
return true; |
||||
} else { |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
/* returns true if reclaimation is proceeding */ |
||||
static bool bpreclaim(grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota, |
||||
bool destructive) { |
||||
if (resource_quota->reclaiming) return true; |
||||
grpc_bulist list = destructive ? GRPC_BULIST_RECLAIMER_DESTRUCTIVE |
||||
: GRPC_BULIST_RECLAIMER_BENIGN; |
||||
grpc_resource_user *resource_user = bulist_pop(resource_quota, list); |
||||
if (resource_user == NULL) return false; |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: initiate %s reclaimation", resource_quota->name, |
||||
resource_user->name, destructive ? "destructive" : "benign"); |
||||
} |
||||
resource_quota->reclaiming = true; |
||||
grpc_resource_quota_internal_ref(resource_quota); |
||||
grpc_closure *c = resource_user->reclaimers[destructive]; |
||||
resource_user->reclaimers[destructive] = NULL; |
||||
grpc_closure_run(exec_ctx, c, GRPC_ERROR_NONE); |
||||
return true; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* bu_slice: a slice implementation that is backed by a grpc_resource_user |
||||
*/ |
||||
|
||||
typedef struct { |
||||
gpr_slice_refcount base; |
||||
gpr_refcount refs; |
||||
grpc_resource_user *resource_user; |
||||
size_t size; |
||||
} bu_slice_refcount; |
||||
|
||||
static void bu_slice_ref(void *p) { |
||||
bu_slice_refcount *rc = p; |
||||
gpr_ref(&rc->refs); |
||||
} |
||||
|
||||
static void bu_slice_unref(void *p) { |
||||
bu_slice_refcount *rc = p; |
||||
if (gpr_unref(&rc->refs)) { |
||||
/* TODO(ctiller): this is dangerous, but I think safe for now:
|
||||
we have no guarantee here that we're at a safe point for creating an |
||||
execution context, but we have no way of writing this code otherwise. |
||||
In the future: consider lifting gpr_slice to grpc, and offering an |
||||
internal_{ref,unref} pair that is execution context aware. Alternatively, |
||||
make exec_ctx be thread local and 'do the right thing' (whatever that is) |
||||
if NULL */ |
||||
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
||||
grpc_resource_user_free(&exec_ctx, rc->resource_user, rc->size); |
||||
grpc_exec_ctx_finish(&exec_ctx); |
||||
gpr_free(rc); |
||||
} |
||||
} |
||||
|
||||
static gpr_slice bu_slice_create(grpc_resource_user *resource_user, size_t size) { |
||||
bu_slice_refcount *rc = gpr_malloc(sizeof(bu_slice_refcount) + size); |
||||
rc->base.ref = bu_slice_ref; |
||||
rc->base.unref = bu_slice_unref; |
||||
gpr_ref_init(&rc->refs, 1); |
||||
rc->resource_user = resource_user; |
||||
rc->size = size; |
||||
gpr_slice slice; |
||||
slice.refcount = &rc->base; |
||||
slice.data.refcounted.bytes = (uint8_t *)(rc + 1); |
||||
slice.data.refcounted.length = size; |
||||
return slice; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_resource_quota internal implementation |
||||
*/ |
||||
|
||||
static void bu_allocate(grpc_exec_ctx *exec_ctx, void *bu, grpc_error *error) { |
||||
grpc_resource_user *resource_user = bu; |
||||
if (bulist_empty(resource_user->resource_quota, GRPC_BULIST_AWAITING_ALLOCATION)) { |
||||
bpstep_sched(exec_ctx, resource_user->resource_quota); |
||||
} |
||||
bulist_add_tail(resource_user, GRPC_BULIST_AWAITING_ALLOCATION); |
||||
} |
||||
|
||||
static void bu_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *bu, |
||||
grpc_error *error) { |
||||
grpc_resource_user *resource_user = bu; |
||||
if (!bulist_empty(resource_user->resource_quota, |
||||
GRPC_BULIST_AWAITING_ALLOCATION) && |
||||
bulist_empty(resource_user->resource_quota, GRPC_BULIST_NON_EMPTY_FREE_POOL)) { |
||||
bpstep_sched(exec_ctx, resource_user->resource_quota); |
||||
} |
||||
bulist_add_tail(resource_user, GRPC_BULIST_NON_EMPTY_FREE_POOL); |
||||
} |
||||
|
||||
static void bu_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
||||
grpc_error *error) { |
||||
grpc_resource_user *resource_user = bu; |
||||
if (!bulist_empty(resource_user->resource_quota, |
||||
GRPC_BULIST_AWAITING_ALLOCATION) && |
||||
bulist_empty(resource_user->resource_quota, GRPC_BULIST_NON_EMPTY_FREE_POOL) && |
||||
bulist_empty(resource_user->resource_quota, GRPC_BULIST_RECLAIMER_BENIGN)) { |
||||
bpstep_sched(exec_ctx, resource_user->resource_quota); |
||||
} |
||||
bulist_add_tail(resource_user, GRPC_BULIST_RECLAIMER_BENIGN); |
||||
} |
||||
|
||||
static void bu_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *bu, |
||||
grpc_error *error) { |
||||
grpc_resource_user *resource_user = bu; |
||||
if (!bulist_empty(resource_user->resource_quota, |
||||
GRPC_BULIST_AWAITING_ALLOCATION) && |
||||
bulist_empty(resource_user->resource_quota, GRPC_BULIST_NON_EMPTY_FREE_POOL) && |
||||
bulist_empty(resource_user->resource_quota, GRPC_BULIST_RECLAIMER_BENIGN) && |
||||
bulist_empty(resource_user->resource_quota, |
||||
GRPC_BULIST_RECLAIMER_DESTRUCTIVE)) { |
||||
bpstep_sched(exec_ctx, resource_user->resource_quota); |
||||
} |
||||
bulist_add_tail(resource_user, GRPC_BULIST_RECLAIMER_DESTRUCTIVE); |
||||
} |
||||
|
||||
static void bu_destroy(grpc_exec_ctx *exec_ctx, void *bu, grpc_error *error) { |
||||
grpc_resource_user *resource_user = bu; |
||||
GPR_ASSERT(resource_user->allocated == 0); |
||||
for (int i = 0; i < GRPC_BULIST_COUNT; i++) { |
||||
bulist_remove(resource_user, (grpc_bulist)i); |
||||
} |
||||
grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[0], |
||||
GRPC_ERROR_CANCELLED, NULL); |
||||
grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[1], |
||||
GRPC_ERROR_CANCELLED, NULL); |
||||
grpc_exec_ctx_sched(exec_ctx, (grpc_closure *)gpr_atm_no_barrier_load( |
||||
&resource_user->on_done_destroy_closure), |
||||
GRPC_ERROR_NONE, NULL); |
||||
if (resource_user->free_pool != 0) { |
||||
resource_user->resource_quota->free_pool += resource_user->free_pool; |
||||
bpstep_sched(exec_ctx, resource_user->resource_quota); |
||||
} |
||||
} |
||||
|
||||
static void bu_allocated_slices(grpc_exec_ctx *exec_ctx, void *ts, |
||||
grpc_error *error) { |
||||
grpc_resource_user_slice_allocator *slice_allocator = ts; |
||||
if (error == GRPC_ERROR_NONE) { |
||||
for (size_t i = 0; i < slice_allocator->count; i++) { |
||||
gpr_slice_buffer_add_indexed(slice_allocator->dest, |
||||
bu_slice_create(slice_allocator->resource_user, |
||||
slice_allocator->length)); |
||||
} |
||||
} |
||||
grpc_closure_run(exec_ctx, &slice_allocator->on_done, GRPC_ERROR_REF(error)); |
||||
} |
||||
|
||||
typedef struct { |
||||
int64_t size; |
||||
grpc_resource_quota *resource_quota; |
||||
grpc_closure closure; |
||||
} bp_resize_args; |
||||
|
||||
static void bp_resize(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) { |
||||
bp_resize_args *a = args; |
||||
int64_t delta = a->size - a->resource_quota->size; |
||||
a->resource_quota->size += delta; |
||||
a->resource_quota->free_pool += delta; |
||||
if (delta < 0 && a->resource_quota->free_pool < 0) { |
||||
bpstep_sched(exec_ctx, a->resource_quota); |
||||
} else if (delta > 0 && |
||||
!bulist_empty(a->resource_quota, GRPC_BULIST_AWAITING_ALLOCATION)) { |
||||
bpstep_sched(exec_ctx, a->resource_quota); |
||||
} |
||||
grpc_resource_quota_internal_unref(exec_ctx, a->resource_quota); |
||||
gpr_free(a); |
||||
} |
||||
|
||||
static void bp_reclaimation_done(grpc_exec_ctx *exec_ctx, void *bp, |
||||
grpc_error *error) { |
||||
grpc_resource_quota *resource_quota = bp; |
||||
resource_quota->reclaiming = false; |
||||
bpstep_sched(exec_ctx, resource_quota); |
||||
grpc_resource_quota_internal_unref(exec_ctx, resource_quota); |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_resource_quota api |
||||
*/ |
||||
|
||||
grpc_resource_quota *grpc_resource_quota_create(const char *name) { |
||||
grpc_resource_quota *resource_quota = gpr_malloc(sizeof(*resource_quota)); |
||||
gpr_ref_init(&resource_quota->refs, 1); |
||||
resource_quota->combiner = grpc_combiner_create(NULL); |
||||
resource_quota->free_pool = INT64_MAX; |
||||
resource_quota->size = INT64_MAX; |
||||
resource_quota->step_scheduled = false; |
||||
resource_quota->reclaiming = false; |
||||
if (name != NULL) { |
||||
resource_quota->name = gpr_strdup(name); |
||||
} else { |
||||
gpr_asprintf(&resource_quota->name, "anonymous_pool_%" PRIxPTR, |
||||
(intptr_t)resource_quota); |
||||
} |
||||
grpc_closure_init(&resource_quota->bpstep_closure, bpstep, resource_quota); |
||||
grpc_closure_init(&resource_quota->bpreclaimation_done_closure, |
||||
bp_reclaimation_done, resource_quota); |
||||
for (int i = 0; i < GRPC_BULIST_COUNT; i++) { |
||||
resource_quota->roots[i] = NULL; |
||||
} |
||||
return resource_quota; |
||||
} |
||||
|
||||
void grpc_resource_quota_internal_unref(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_quota *resource_quota) { |
||||
if (gpr_unref(&resource_quota->refs)) { |
||||
grpc_combiner_destroy(exec_ctx, resource_quota->combiner); |
||||
gpr_free(resource_quota->name); |
||||
gpr_free(resource_quota); |
||||
} |
||||
} |
||||
|
||||
void grpc_resource_quota_unref(grpc_resource_quota *resource_quota) { |
||||
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
||||
grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); |
||||
grpc_exec_ctx_finish(&exec_ctx); |
||||
} |
||||
|
||||
grpc_resource_quota *grpc_resource_quota_internal_ref(grpc_resource_quota *resource_quota) { |
||||
gpr_ref(&resource_quota->refs); |
||||
return resource_quota; |
||||
} |
||||
|
||||
void grpc_resource_quota_ref(grpc_resource_quota *resource_quota) { |
||||
grpc_resource_quota_internal_ref(resource_quota); |
||||
} |
||||
|
||||
void grpc_resource_quota_resize(grpc_resource_quota *resource_quota, size_t size) { |
||||
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; |
||||
bp_resize_args *a = gpr_malloc(sizeof(*a)); |
||||
a->resource_quota = grpc_resource_quota_internal_ref(resource_quota); |
||||
a->size = (int64_t)size; |
||||
grpc_closure_init(&a->closure, bp_resize, a); |
||||
grpc_combiner_execute(&exec_ctx, resource_quota->combiner, &a->closure, |
||||
GRPC_ERROR_NONE, false); |
||||
grpc_exec_ctx_finish(&exec_ctx); |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_resource_user channel args api |
||||
*/ |
||||
|
||||
grpc_resource_quota *grpc_resource_quota_from_channel_args( |
||||
const grpc_channel_args *channel_args) { |
||||
for (size_t i = 0; i < channel_args->num_args; i++) { |
||||
if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_BUFFER_POOL)) { |
||||
if (channel_args->args[i].type == GRPC_ARG_POINTER) { |
||||
return grpc_resource_quota_internal_ref( |
||||
channel_args->args[i].value.pointer.p); |
||||
} else { |
||||
gpr_log(GPR_DEBUG, GRPC_ARG_BUFFER_POOL " should be a pointer"); |
||||
} |
||||
} |
||||
} |
||||
return grpc_resource_quota_create(NULL); |
||||
} |
||||
|
||||
static void *bp_copy(void *bp) { |
||||
grpc_resource_quota_ref(bp); |
||||
return bp; |
||||
} |
||||
|
||||
static void bp_destroy(void *bp) { grpc_resource_quota_unref(bp); } |
||||
|
||||
static int bp_cmp(void *a, void *b) { return GPR_ICMP(a, b); } |
||||
|
||||
const grpc_arg_pointer_vtable *grpc_resource_quota_arg_vtable(void) { |
||||
static const grpc_arg_pointer_vtable vtable = {bp_copy, bp_destroy, bp_cmp}; |
||||
return &vtable; |
||||
} |
||||
|
||||
/*******************************************************************************
|
||||
* grpc_resource_user api |
||||
*/ |
||||
|
||||
void grpc_resource_user_init(grpc_resource_user *resource_user, |
||||
grpc_resource_quota *resource_quota, const char *name) { |
||||
resource_user->resource_quota = grpc_resource_quota_internal_ref(resource_quota); |
||||
grpc_closure_init(&resource_user->allocate_closure, &bu_allocate, resource_user); |
||||
grpc_closure_init(&resource_user->add_to_free_pool_closure, |
||||
&bu_add_to_free_pool, resource_user); |
||||
grpc_closure_init(&resource_user->post_reclaimer_closure[0], |
||||
&bu_post_benign_reclaimer, resource_user); |
||||
grpc_closure_init(&resource_user->post_reclaimer_closure[1], |
||||
&bu_post_destructive_reclaimer, resource_user); |
||||
grpc_closure_init(&resource_user->destroy_closure, &bu_destroy, resource_user); |
||||
gpr_mu_init(&resource_user->mu); |
||||
resource_user->allocated = 0; |
||||
resource_user->free_pool = 0; |
||||
grpc_closure_list_init(&resource_user->on_allocated); |
||||
resource_user->allocating = false; |
||||
resource_user->added_to_free_pool = false; |
||||
gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, 0); |
||||
resource_user->reclaimers[0] = NULL; |
||||
resource_user->reclaimers[1] = NULL; |
||||
for (int i = 0; i < GRPC_BULIST_COUNT; i++) { |
||||
resource_user->links[i].next = resource_user->links[i].prev = NULL; |
||||
} |
||||
#ifndef NDEBUG |
||||
resource_user->asan_canary = gpr_malloc(1); |
||||
#endif |
||||
if (name != NULL) { |
||||
resource_user->name = gpr_strdup(name); |
||||
} else { |
||||
gpr_asprintf(&resource_user->name, "anonymous_resource_user_%" PRIxPTR, |
||||
(intptr_t)resource_user); |
||||
} |
||||
} |
||||
|
||||
void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_user *resource_user, |
||||
grpc_closure *on_done) { |
||||
gpr_mu_lock(&resource_user->mu); |
||||
GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->on_done_destroy_closure) == |
||||
0); |
||||
gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, |
||||
(gpr_atm)on_done); |
||||
if (resource_user->allocated == 0) { |
||||
grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, |
||||
&resource_user->destroy_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
} |
||||
|
||||
void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_user *resource_user) { |
||||
#ifndef NDEBUG |
||||
gpr_free(resource_user->asan_canary); |
||||
#endif |
||||
grpc_resource_quota_internal_unref(exec_ctx, resource_user->resource_quota); |
||||
gpr_mu_destroy(&resource_user->mu); |
||||
gpr_free(resource_user->name); |
||||
} |
||||
|
||||
void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_user *resource_user, size_t size, |
||||
grpc_closure *optional_on_done) { |
||||
gpr_mu_lock(&resource_user->mu); |
||||
grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( |
||||
&resource_user->on_done_destroy_closure); |
||||
if (on_done_destroy != NULL) { |
||||
/* already shutdown */ |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: alloc %" PRIdPTR " after shutdown", |
||||
resource_user->resource_quota->name, resource_user->name, size); |
||||
} |
||||
grpc_exec_ctx_sched( |
||||
exec_ctx, optional_on_done, |
||||
GRPC_ERROR_CREATE("Buffer pool user is already shutdown"), NULL); |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
return; |
||||
} |
||||
resource_user->allocated += (int64_t)size; |
||||
resource_user->free_pool -= (int64_t)size; |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: alloc %" PRIdPTR "; allocated -> %" PRId64 |
||||
", free_pool -> %" PRId64, |
||||
resource_user->resource_quota->name, resource_user->name, size, |
||||
resource_user->allocated, resource_user->free_pool); |
||||
} |
||||
if (resource_user->free_pool < 0) { |
||||
grpc_closure_list_append(&resource_user->on_allocated, optional_on_done, |
||||
GRPC_ERROR_NONE); |
||||
if (!resource_user->allocating) { |
||||
resource_user->allocating = true; |
||||
grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, |
||||
&resource_user->allocate_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
} else { |
||||
grpc_exec_ctx_sched(exec_ctx, optional_on_done, GRPC_ERROR_NONE, NULL); |
||||
} |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
} |
||||
|
||||
void grpc_resource_user_free(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_user *resource_user, size_t size) { |
||||
gpr_mu_lock(&resource_user->mu); |
||||
GPR_ASSERT(resource_user->allocated >= (int64_t)size); |
||||
bool was_zero_or_negative = resource_user->free_pool <= 0; |
||||
resource_user->free_pool += (int64_t)size; |
||||
resource_user->allocated -= (int64_t)size; |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: free %" PRIdPTR "; allocated -> %" PRId64 |
||||
", free_pool -> %" PRId64, |
||||
resource_user->resource_quota->name, resource_user->name, size, |
||||
resource_user->allocated, resource_user->free_pool); |
||||
} |
||||
bool is_bigger_than_zero = resource_user->free_pool > 0; |
||||
if (is_bigger_than_zero && was_zero_or_negative && |
||||
!resource_user->added_to_free_pool) { |
||||
resource_user->added_to_free_pool = true; |
||||
grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, |
||||
&resource_user->add_to_free_pool_closure, |
||||
GRPC_ERROR_NONE, false); |
||||
} |
||||
grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load( |
||||
&resource_user->on_done_destroy_closure); |
||||
if (on_done_destroy != NULL && resource_user->allocated == 0) { |
||||
grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, |
||||
&resource_user->destroy_closure, GRPC_ERROR_NONE, |
||||
false); |
||||
} |
||||
gpr_mu_unlock(&resource_user->mu); |
||||
} |
||||
|
||||
void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_user *resource_user, |
||||
bool destructive, grpc_closure *closure) { |
||||
if (gpr_atm_acq_load(&resource_user->on_done_destroy_closure) == 0) { |
||||
GPR_ASSERT(resource_user->reclaimers[destructive] == NULL); |
||||
resource_user->reclaimers[destructive] = closure; |
||||
grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, |
||||
&resource_user->post_reclaimer_closure[destructive], |
||||
GRPC_ERROR_NONE, false); |
||||
} else { |
||||
grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL); |
||||
} |
||||
} |
||||
|
||||
void grpc_resource_user_finish_reclaimation(grpc_exec_ctx *exec_ctx, |
||||
grpc_resource_user *resource_user) { |
||||
if (grpc_resource_quota_trace) { |
||||
gpr_log(GPR_DEBUG, "BP %s %s: reclaimation complete", |
||||
resource_user->resource_quota->name, resource_user->name); |
||||
} |
||||
grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner, |
||||
&resource_user->resource_quota->bpreclaimation_done_closure, |
||||
GRPC_ERROR_NONE, false); |
||||
} |
||||
|
||||
void grpc_resource_user_slice_allocator_init( |
||||
grpc_resource_user_slice_allocator *slice_allocator, |
||||
grpc_resource_user *resource_user, grpc_iomgr_cb_func cb, void *p) { |
||||
grpc_closure_init(&slice_allocator->on_allocated, bu_allocated_slices, |
||||
slice_allocator); |
||||
grpc_closure_init(&slice_allocator->on_done, cb, p); |
||||
slice_allocator->resource_user = resource_user; |
||||
} |
||||
|
||||
void grpc_resource_user_alloc_slices( |
||||
grpc_exec_ctx *exec_ctx, grpc_resource_user_slice_allocator *slice_allocator, |
||||
size_t length, size_t count, gpr_slice_buffer *dest) { |
||||
slice_allocator->length = length; |
||||
slice_allocator->count = count; |
||||
slice_allocator->dest = dest; |
||||
grpc_resource_user_alloc(exec_ctx, slice_allocator->resource_user, count * length, |
||||
&slice_allocator->on_allocated); |
||||
} |
||||
@ -0,0 +1,352 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2016, Google Inc. |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or without |
||||
* modification, are permitted provided that the following conditions are |
||||
* met: |
||||
* |
||||
* * Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* * Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following disclaimer |
||||
* in the documentation and/or other materials provided with the |
||||
* distribution. |
||||
* * Neither the name of Google Inc. nor the names of its |
||||
* contributors may be used to endorse or promote products derived from |
||||
* this software without specific prior written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
* |
||||
*/ |
||||
|
||||
#include "test/core/end2end/end2end_tests.h" |
||||
|
||||
#include <stdio.h> |
||||
#include <string.h> |
||||
|
||||
#include <grpc/byte_buffer.h> |
||||
#include <grpc/support/alloc.h> |
||||
#include <grpc/support/log.h> |
||||
#include <grpc/support/time.h> |
||||
#include <grpc/support/useful.h> |
||||
#include "test/core/end2end/cq_verifier.h" |
||||
|
||||
static void *tag(intptr_t t) { return (void *)t; } |
||||
|
||||
static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, |
||||
const char *test_name, |
||||
grpc_channel_args *client_args, |
||||
grpc_channel_args *server_args) { |
||||
grpc_end2end_test_fixture f; |
||||
gpr_log(GPR_INFO, "%s/%s", test_name, config.name); |
||||
f = config.create_fixture(client_args, server_args); |
||||
config.init_server(&f, server_args); |
||||
config.init_client(&f, client_args); |
||||
return f; |
||||
} |
||||
|
||||
static gpr_timespec n_seconds_time(int n) { |
||||
return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n); |
||||
} |
||||
|
||||
static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } |
||||
|
||||
static void drain_cq(grpc_completion_queue *cq) { |
||||
grpc_event ev; |
||||
do { |
||||
ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); |
||||
} while (ev.type != GRPC_QUEUE_SHUTDOWN); |
||||
} |
||||
|
||||
static void shutdown_server(grpc_end2end_test_fixture *f) { |
||||
if (!f->server) return; |
||||
grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); |
||||
GPR_ASSERT(grpc_completion_queue_pluck( |
||||
f->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL) |
||||
.type == GRPC_OP_COMPLETE); |
||||
grpc_server_destroy(f->server); |
||||
f->server = NULL; |
||||
} |
||||
|
||||
static void shutdown_client(grpc_end2end_test_fixture *f) { |
||||
if (!f->client) return; |
||||
grpc_channel_destroy(f->client); |
||||
f->client = NULL; |
||||
} |
||||
|
||||
static void end_test(grpc_end2end_test_fixture *f) { |
||||
shutdown_server(f); |
||||
shutdown_client(f); |
||||
|
||||
grpc_completion_queue_shutdown(f->cq); |
||||
drain_cq(f->cq); |
||||
grpc_completion_queue_destroy(f->cq); |
||||
} |
||||
|
||||
/* Creates and returns a gpr_slice containing random alphanumeric characters. */ |
||||
static gpr_slice generate_random_slice() { |
||||
size_t i; |
||||
static const char chars[] = "abcdefghijklmnopqrstuvwxyz1234567890"; |
||||
char output[1024 * 1024]; |
||||
for (i = 0; i < GPR_ARRAY_SIZE(output) - 1; ++i) { |
||||
output[i] = chars[rand() % (int)(sizeof(chars) - 1)]; |
||||
} |
||||
output[GPR_ARRAY_SIZE(output) - 1] = '\0'; |
||||
return gpr_slice_from_copied_string(output); |
||||
} |
||||
|
||||
void resource_quota_server(grpc_end2end_test_config config) { |
||||
grpc_resource_quota *resource_quota = grpc_resource_quota_create("test_server"); |
||||
grpc_resource_quota_resize(resource_quota, 5 * 1024 * 1024); |
||||
|
||||
#define NUM_CALLS 100 |
||||
#define CLIENT_BASE_TAG 1000 |
||||
#define SERVER_START_BASE_TAG 2000 |
||||
#define SERVER_RECV_BASE_TAG 3000 |
||||
#define SERVER_END_BASE_TAG 4000 |
||||
|
||||
grpc_arg arg; |
||||
arg.key = GRPC_ARG_BUFFER_POOL; |
||||
arg.type = GRPC_ARG_POINTER; |
||||
arg.value.pointer.p = resource_quota; |
||||
arg.value.pointer.vtable = grpc_resource_quota_arg_vtable(); |
||||
grpc_channel_args args = {1, &arg}; |
||||
|
||||
grpc_end2end_test_fixture f = |
||||
begin_test(config, "resource_quota_server", NULL, &args); |
||||
|
||||
/* Create large request and response bodies. These are big enough to require
|
||||
* multiple round trips to deliver to the peer, and their exact contents of |
||||
* will be verified on completion. */ |
||||
gpr_slice request_payload_slice = generate_random_slice(); |
||||
|
||||
grpc_call *client_calls[NUM_CALLS]; |
||||
grpc_call *server_calls[NUM_CALLS]; |
||||
grpc_metadata_array initial_metadata_recv[NUM_CALLS]; |
||||
grpc_metadata_array trailing_metadata_recv[NUM_CALLS]; |
||||
grpc_metadata_array request_metadata_recv[NUM_CALLS]; |
||||
grpc_call_details call_details[NUM_CALLS]; |
||||
grpc_status_code status[NUM_CALLS]; |
||||
char *details[NUM_CALLS]; |
||||
size_t details_capacity[NUM_CALLS]; |
||||
grpc_byte_buffer *request_payload_recv[NUM_CALLS]; |
||||
int was_cancelled[NUM_CALLS]; |
||||
grpc_call_error error; |
||||
int pending_client_calls = 0; |
||||
int pending_server_start_calls = 0; |
||||
int pending_server_recv_calls = 0; |
||||
int pending_server_end_calls = 0; |
||||
int cancelled_calls_on_client = 0; |
||||
int cancelled_calls_on_server = 0; |
||||
|
||||
grpc_byte_buffer *request_payload = |
||||
grpc_raw_byte_buffer_create(&request_payload_slice, 1); |
||||
|
||||
grpc_op ops[6]; |
||||
grpc_op *op; |
||||
|
||||
for (int i = 0; i < NUM_CALLS; i++) { |
||||
grpc_metadata_array_init(&initial_metadata_recv[i]); |
||||
grpc_metadata_array_init(&trailing_metadata_recv[i]); |
||||
grpc_metadata_array_init(&request_metadata_recv[i]); |
||||
grpc_call_details_init(&call_details[i]); |
||||
details[i] = NULL; |
||||
details_capacity[i] = 0; |
||||
request_payload_recv[i] = NULL; |
||||
was_cancelled[i] = 0; |
||||
} |
||||
|
||||
for (int i = 0; i < NUM_CALLS; i++) { |
||||
error = grpc_server_request_call( |
||||
f.server, &server_calls[i], &call_details[i], &request_metadata_recv[i], |
||||
f.cq, f.cq, tag(SERVER_START_BASE_TAG + i)); |
||||
GPR_ASSERT(GRPC_CALL_OK == error); |
||||
|
||||
pending_server_start_calls++; |
||||
} |
||||
|
||||
for (int i = 0; i < NUM_CALLS; i++) { |
||||
client_calls[i] = grpc_channel_create_call( |
||||
f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, "/foo", |
||||
"foo.test.google.fr", n_seconds_time(60), NULL); |
||||
|
||||
memset(ops, 0, sizeof(ops)); |
||||
op = ops; |
||||
op->op = GRPC_OP_SEND_INITIAL_METADATA; |
||||
op->data.send_initial_metadata.count = 0; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
op->op = GRPC_OP_SEND_MESSAGE; |
||||
op->data.send_message = request_payload; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
op->op = GRPC_OP_RECV_INITIAL_METADATA; |
||||
op->data.recv_initial_metadata = &initial_metadata_recv[i]; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; |
||||
op->data.recv_status_on_client.trailing_metadata = |
||||
&trailing_metadata_recv[i]; |
||||
op->data.recv_status_on_client.status = &status[i]; |
||||
op->data.recv_status_on_client.status_details = &details[i]; |
||||
op->data.recv_status_on_client.status_details_capacity = |
||||
&details_capacity[i]; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
error = grpc_call_start_batch(client_calls[i], ops, (size_t)(op - ops), |
||||
tag(CLIENT_BASE_TAG + i), NULL); |
||||
GPR_ASSERT(GRPC_CALL_OK == error); |
||||
|
||||
pending_client_calls++; |
||||
} |
||||
|
||||
while (pending_client_calls + pending_server_recv_calls + |
||||
pending_server_end_calls > |
||||
0) { |
||||
grpc_event ev = grpc_completion_queue_next(f.cq, n_seconds_time(10), NULL); |
||||
GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); |
||||
|
||||
int ev_tag = (int)(intptr_t)ev.tag; |
||||
if (ev_tag < CLIENT_BASE_TAG) { |
||||
abort(); /* illegal tag */ |
||||
} else if (ev_tag < SERVER_START_BASE_TAG) { |
||||
/* client call finished */ |
||||
int call_id = ev_tag - CLIENT_BASE_TAG; |
||||
GPR_ASSERT(call_id >= 0); |
||||
GPR_ASSERT(call_id < NUM_CALLS); |
||||
switch (status[call_id]) { |
||||
case GRPC_STATUS_RESOURCE_EXHAUSTED: |
||||
cancelled_calls_on_client++; |
||||
break; |
||||
case GRPC_STATUS_OK: |
||||
break; |
||||
default: |
||||
gpr_log(GPR_ERROR, "Unexpected status code: %d", status[call_id]); |
||||
abort(); |
||||
} |
||||
GPR_ASSERT(pending_client_calls > 0); |
||||
|
||||
grpc_metadata_array_destroy(&initial_metadata_recv[call_id]); |
||||
grpc_metadata_array_destroy(&trailing_metadata_recv[call_id]); |
||||
grpc_call_destroy(client_calls[call_id]); |
||||
gpr_free(details[call_id]); |
||||
|
||||
pending_client_calls--; |
||||
} else if (ev_tag < SERVER_RECV_BASE_TAG) { |
||||
/* new incoming call to the server */ |
||||
int call_id = ev_tag - SERVER_START_BASE_TAG; |
||||
GPR_ASSERT(call_id >= 0); |
||||
GPR_ASSERT(call_id < NUM_CALLS); |
||||
|
||||
memset(ops, 0, sizeof(ops)); |
||||
op = ops; |
||||
op->op = GRPC_OP_SEND_INITIAL_METADATA; |
||||
op->data.send_initial_metadata.count = 0; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
op->op = GRPC_OP_RECV_MESSAGE; |
||||
op->data.recv_message = &request_payload_recv[call_id]; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
error = |
||||
grpc_call_start_batch(server_calls[call_id], ops, (size_t)(op - ops), |
||||
tag(SERVER_RECV_BASE_TAG + call_id), NULL); |
||||
GPR_ASSERT(GRPC_CALL_OK == error); |
||||
|
||||
GPR_ASSERT(pending_server_start_calls > 0); |
||||
pending_server_start_calls--; |
||||
pending_server_recv_calls++; |
||||
|
||||
grpc_call_details_destroy(&call_details[call_id]); |
||||
grpc_metadata_array_destroy(&request_metadata_recv[call_id]); |
||||
} else if (ev_tag < SERVER_END_BASE_TAG) { |
||||
/* finished read on the server */ |
||||
int call_id = ev_tag - SERVER_RECV_BASE_TAG; |
||||
GPR_ASSERT(call_id >= 0); |
||||
GPR_ASSERT(call_id < NUM_CALLS); |
||||
|
||||
if (ev.success) { |
||||
if (request_payload_recv[call_id] != NULL) { |
||||
grpc_byte_buffer_destroy(request_payload_recv[call_id]); |
||||
request_payload_recv[call_id] = NULL; |
||||
} |
||||
} else { |
||||
GPR_ASSERT(request_payload_recv[call_id] == NULL); |
||||
} |
||||
|
||||
memset(ops, 0, sizeof(ops)); |
||||
op = ops; |
||||
op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; |
||||
op->data.recv_close_on_server.cancelled = &was_cancelled[call_id]; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; |
||||
op->data.send_status_from_server.trailing_metadata_count = 0; |
||||
op->data.send_status_from_server.status = GRPC_STATUS_OK; |
||||
op->data.send_status_from_server.status_details = "xyz"; |
||||
op->flags = 0; |
||||
op->reserved = NULL; |
||||
op++; |
||||
error = |
||||
grpc_call_start_batch(server_calls[call_id], ops, (size_t)(op - ops), |
||||
tag(SERVER_END_BASE_TAG + call_id), NULL); |
||||
GPR_ASSERT(GRPC_CALL_OK == error); |
||||
|
||||
GPR_ASSERT(pending_server_recv_calls > 0); |
||||
pending_server_recv_calls--; |
||||
pending_server_end_calls++; |
||||
} else { |
||||
int call_id = ev_tag - SERVER_END_BASE_TAG; |
||||
GPR_ASSERT(call_id >= 0); |
||||
GPR_ASSERT(call_id < NUM_CALLS); |
||||
|
||||
if (was_cancelled[call_id]) { |
||||
cancelled_calls_on_server++; |
||||
} |
||||
GPR_ASSERT(pending_server_end_calls > 0); |
||||
pending_server_end_calls--; |
||||
|
||||
grpc_call_destroy(server_calls[call_id]); |
||||
} |
||||
} |
||||
|
||||
gpr_log( |
||||
GPR_INFO, |
||||
"Done. %d total calls: %d cancelled at server, %d cancelled at client.", |
||||
NUM_CALLS, cancelled_calls_on_server, cancelled_calls_on_client); |
||||
|
||||
GPR_ASSERT(cancelled_calls_on_client >= cancelled_calls_on_server); |
||||
GPR_ASSERT(cancelled_calls_on_server >= 0.9 * cancelled_calls_on_client); |
||||
|
||||
grpc_byte_buffer_destroy(request_payload); |
||||
gpr_slice_unref(request_payload_slice); |
||||
grpc_resource_quota_unref(resource_quota); |
||||
|
||||
end_test(&f); |
||||
config.tear_down_data(&f); |
||||
} |
||||
|
||||
void resource_quota_server_pre_init(void) {} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,20 +1,20 @@ |
||||
<?xml version="1.0" encoding="utf-8"?> |
||||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> |
||||
<ItemGroup> |
||||
<ClCompile Include="$(SolutionDir)\..\test\core\iomgr\buffer_pool_test.c"> |
||||
<ClCompile Include="$(SolutionDir)\..\test\core\iomgr\resource_quota_test.c"> |
||||
<Filter>test\core\iomgr</Filter> |
||||
</ClCompile> |
||||
</ItemGroup> |
||||
|
||||
<ItemGroup> |
||||
<Filter Include="test"> |
||||
<UniqueIdentifier>{94e599c3-a059-4581-0cac-d15361ec8a7d}</UniqueIdentifier> |
||||
<UniqueIdentifier>{06783e32-dbf0-7e7c-7b50-12b278f9cc12}</UniqueIdentifier> |
||||
</Filter> |
||||
<Filter Include="test\core"> |
||||
<UniqueIdentifier>{6d25d413-0043-5a1c-52f7-7d25809be372}</UniqueIdentifier> |
||||
<UniqueIdentifier>{c4f432b6-577b-e3ed-fec9-a915af5ebbd5}</UniqueIdentifier> |
||||
</Filter> |
||||
<Filter Include="test\core\iomgr"> |
||||
<UniqueIdentifier>{64b38e90-4497-be2e-cee1-402590cbea8a}</UniqueIdentifier> |
||||
<UniqueIdentifier>{fcc82d68-ffb2-0843-83aa-175006c43aeb}</UniqueIdentifier> |
||||
</Filter> |
||||
</ItemGroup> |
||||
</Project> |
||||
Loading…
Reference in new issue