|
|
|
|
@ -68,8 +68,8 @@ void grpc_mdelem_trace_ref(void* md, const grpc_slice& key, |
|
|
|
|
char* key_str = grpc_slice_to_c_string(key); |
|
|
|
|
char* value_str = grpc_slice_to_c_string(value); |
|
|
|
|
gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, |
|
|
|
|
"ELM REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", md, refcnt, |
|
|
|
|
refcnt + 1, key_str, value_str); |
|
|
|
|
"mdelem REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", md, |
|
|
|
|
refcnt, refcnt + 1, key_str, value_str); |
|
|
|
|
gpr_free(key_str); |
|
|
|
|
gpr_free(value_str); |
|
|
|
|
} |
|
|
|
|
@ -82,7 +82,7 @@ void grpc_mdelem_trace_unref(void* md, const grpc_slice& key, |
|
|
|
|
char* key_str = grpc_slice_to_c_string(key); |
|
|
|
|
char* value_str = grpc_slice_to_c_string(value); |
|
|
|
|
gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, |
|
|
|
|
"ELM UNREF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", md, |
|
|
|
|
"mdelem UNREF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", md, |
|
|
|
|
refcnt, refcnt - 1, key_str, value_str); |
|
|
|
|
gpr_free(key_str); |
|
|
|
|
gpr_free(value_str); |
|
|
|
|
@ -112,14 +112,33 @@ AllocatedMetadata::AllocatedMetadata(const grpc_slice& key, |
|
|
|
|
: RefcountedMdBase(grpc_slice_ref_internal(key), |
|
|
|
|
grpc_slice_ref_internal(value)) { |
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
if (grpc_trace_metadata.enabled()) { |
|
|
|
|
char* key_str = grpc_slice_to_c_string(key); |
|
|
|
|
char* value_str = grpc_slice_to_c_string(value); |
|
|
|
|
gpr_log(GPR_DEBUG, "ELM ALLOC:%p:%" PRIdPTR ": '%s' = '%s'", this, |
|
|
|
|
RefValue(), key_str, value_str); |
|
|
|
|
gpr_free(key_str); |
|
|
|
|
gpr_free(value_str); |
|
|
|
|
} |
|
|
|
|
TraceAtStart("ALLOC_MD"); |
|
|
|
|
#endif |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
AllocatedMetadata::AllocatedMetadata(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value, const NoRefKey*) |
|
|
|
|
: RefcountedMdBase(key, grpc_slice_ref_internal(value)) { |
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
TraceAtStart("ALLOC_MD_NOREF_KEY"); |
|
|
|
|
#endif |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
AllocatedMetadata::AllocatedMetadata( |
|
|
|
|
const grpc_core::ManagedMemorySlice& key, |
|
|
|
|
const grpc_core::UnmanagedMemorySlice& value) |
|
|
|
|
: RefcountedMdBase(key, value) { |
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
TraceAtStart("ALLOC_MD_NOREF_KEY_VAL"); |
|
|
|
|
#endif |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
AllocatedMetadata::AllocatedMetadata( |
|
|
|
|
const grpc_core::ExternallyManagedSlice& key, |
|
|
|
|
const grpc_core::UnmanagedMemorySlice& value) |
|
|
|
|
: RefcountedMdBase(key, value) { |
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
TraceAtStart("ALLOC_MD_NOREF_KEY_VAL"); |
|
|
|
|
#endif |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@ -134,6 +153,19 @@ AllocatedMetadata::~AllocatedMetadata() { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
void grpc_core::RefcountedMdBase::TraceAtStart(const char* tag) { |
|
|
|
|
if (grpc_trace_metadata.enabled()) { |
|
|
|
|
char* key_str = grpc_slice_to_c_string(key()); |
|
|
|
|
char* value_str = grpc_slice_to_c_string(value()); |
|
|
|
|
gpr_log(GPR_DEBUG, "mdelem %s:%p:%" PRIdPTR ": '%s' = '%s'", tag, this, |
|
|
|
|
RefValue(), key_str, value_str); |
|
|
|
|
gpr_free(key_str); |
|
|
|
|
gpr_free(value_str); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
#endif |
|
|
|
|
|
|
|
|
|
InternedMetadata::InternedMetadata(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value, uint32_t hash, |
|
|
|
|
InternedMetadata* next) |
|
|
|
|
@ -141,14 +173,16 @@ InternedMetadata::InternedMetadata(const grpc_slice& key, |
|
|
|
|
grpc_slice_ref_internal(value), hash), |
|
|
|
|
link_(next) { |
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
if (grpc_trace_metadata.enabled()) { |
|
|
|
|
char* key_str = grpc_slice_to_c_string(key); |
|
|
|
|
char* value_str = grpc_slice_to_c_string(value); |
|
|
|
|
gpr_log(GPR_DEBUG, "ELM NEW:%p:%" PRIdPTR ": '%s' = '%s'", this, |
|
|
|
|
RefValue(), key_str, value_str); |
|
|
|
|
gpr_free(key_str); |
|
|
|
|
gpr_free(value_str); |
|
|
|
|
} |
|
|
|
|
TraceAtStart("INTERNED_MD"); |
|
|
|
|
#endif |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
InternedMetadata::InternedMetadata(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value, uint32_t hash, |
|
|
|
|
InternedMetadata* next, const NoRefKey*) |
|
|
|
|
: RefcountedMdBase(key, grpc_slice_ref_internal(value), hash), link_(next) { |
|
|
|
|
#ifndef NDEBUG |
|
|
|
|
TraceAtStart("INTERNED_MD_NOREF_KEY"); |
|
|
|
|
#endif |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@ -248,8 +282,8 @@ void InternedMetadata::RefWithShardLocked(mdtab_shard* shard) { |
|
|
|
|
char* value_str = grpc_slice_to_c_string(value()); |
|
|
|
|
intptr_t value = RefValue(); |
|
|
|
|
gpr_log(__FILE__, __LINE__, GPR_LOG_SEVERITY_DEBUG, |
|
|
|
|
"ELM REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", this, value, |
|
|
|
|
value + 1, key_str, value_str); |
|
|
|
|
"mdelem REF:%p:%" PRIdPTR "->%" PRIdPTR ": '%s' = '%s'", this, |
|
|
|
|
value, value + 1, key_str, value_str); |
|
|
|
|
gpr_free(key_str); |
|
|
|
|
gpr_free(value_str); |
|
|
|
|
} |
|
|
|
|
@ -307,36 +341,100 @@ static void rehash_mdtab(mdtab_shard* shard) { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_create( |
|
|
|
|
template <bool key_definitely_static, bool value_definitely_static = false> |
|
|
|
|
static grpc_mdelem md_create_maybe_static(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value); |
|
|
|
|
template <bool key_definitely_static> |
|
|
|
|
static grpc_mdelem md_create_must_intern(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value, |
|
|
|
|
uint32_t hash); |
|
|
|
|
|
|
|
|
|
template <bool key_definitely_static, bool value_definitely_static = false> |
|
|
|
|
static grpc_mdelem md_create( |
|
|
|
|
const grpc_slice& key, const grpc_slice& value, |
|
|
|
|
grpc_mdelem_data* compatible_external_backing_store) { |
|
|
|
|
// Ensure slices are, in fact, static if we claimed they were.
|
|
|
|
|
GPR_DEBUG_ASSERT(!key_definitely_static || |
|
|
|
|
GRPC_IS_STATIC_METADATA_STRING(key)); |
|
|
|
|
GPR_DEBUG_ASSERT(!value_definitely_static || |
|
|
|
|
GRPC_IS_STATIC_METADATA_STRING(value)); |
|
|
|
|
const bool key_is_interned = |
|
|
|
|
key_definitely_static || grpc_slice_is_interned(key); |
|
|
|
|
const bool value_is_interned = |
|
|
|
|
value_definitely_static || grpc_slice_is_interned(value); |
|
|
|
|
// External storage if either slice is not interned and the caller already
|
|
|
|
|
// created a backing store. If no backing store, we allocate one.
|
|
|
|
|
if (!grpc_slice_is_interned(key) || !grpc_slice_is_interned(value)) { |
|
|
|
|
if (!key_is_interned || !value_is_interned) { |
|
|
|
|
if (compatible_external_backing_store != nullptr) { |
|
|
|
|
// Caller provided backing store.
|
|
|
|
|
return GRPC_MAKE_MDELEM(compatible_external_backing_store, |
|
|
|
|
GRPC_MDELEM_STORAGE_EXTERNAL); |
|
|
|
|
} else { |
|
|
|
|
// We allocate backing store.
|
|
|
|
|
return GRPC_MAKE_MDELEM(grpc_core::New<AllocatedMetadata>(key, value), |
|
|
|
|
GRPC_MDELEM_STORAGE_ALLOCATED); |
|
|
|
|
return key_definitely_static |
|
|
|
|
? GRPC_MAKE_MDELEM( |
|
|
|
|
grpc_core::New<AllocatedMetadata>( |
|
|
|
|
key, value, |
|
|
|
|
static_cast<const AllocatedMetadata::NoRefKey*>( |
|
|
|
|
nullptr)), |
|
|
|
|
GRPC_MDELEM_STORAGE_ALLOCATED) |
|
|
|
|
: GRPC_MAKE_MDELEM( |
|
|
|
|
grpc_core::New<AllocatedMetadata>(key, value), |
|
|
|
|
GRPC_MDELEM_STORAGE_ALLOCATED); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return md_create_maybe_static<key_definitely_static, value_definitely_static>( |
|
|
|
|
key, value); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
template <bool key_definitely_static, bool value_definitely_static> |
|
|
|
|
static grpc_mdelem md_create_maybe_static(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value) { |
|
|
|
|
// Ensure slices are, in fact, static if we claimed they were.
|
|
|
|
|
GPR_DEBUG_ASSERT(!key_definitely_static || |
|
|
|
|
GRPC_IS_STATIC_METADATA_STRING(key)); |
|
|
|
|
GPR_DEBUG_ASSERT(!value_definitely_static || |
|
|
|
|
GRPC_IS_STATIC_METADATA_STRING(value)); |
|
|
|
|
GPR_DEBUG_ASSERT(key.refcount != nullptr); |
|
|
|
|
GPR_DEBUG_ASSERT(value.refcount != nullptr); |
|
|
|
|
|
|
|
|
|
const bool key_is_static_mdstr = |
|
|
|
|
key_definitely_static || |
|
|
|
|
key.refcount->GetType() == grpc_slice_refcount::Type::STATIC; |
|
|
|
|
const bool value_is_static_mdstr = |
|
|
|
|
value_definitely_static || |
|
|
|
|
value.refcount->GetType() == grpc_slice_refcount::Type::STATIC; |
|
|
|
|
|
|
|
|
|
const intptr_t kidx = GRPC_STATIC_METADATA_INDEX(key); |
|
|
|
|
|
|
|
|
|
// Not all static slice input yields a statically stored metadata element.
|
|
|
|
|
// It may be worth documenting why.
|
|
|
|
|
if (GRPC_IS_STATIC_METADATA_STRING(key) && |
|
|
|
|
GRPC_IS_STATIC_METADATA_STRING(value)) { |
|
|
|
|
if (key_is_static_mdstr && value_is_static_mdstr) { |
|
|
|
|
grpc_mdelem static_elem = grpc_static_mdelem_for_static_strings( |
|
|
|
|
GRPC_STATIC_METADATA_INDEX(key), GRPC_STATIC_METADATA_INDEX(value)); |
|
|
|
|
kidx, GRPC_STATIC_METADATA_INDEX(value)); |
|
|
|
|
if (!GRPC_MDISNULL(static_elem)) { |
|
|
|
|
return static_elem; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
uint32_t hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash_refcounted(key), |
|
|
|
|
grpc_slice_hash_refcounted(value)); |
|
|
|
|
uint32_t khash = key_definitely_static |
|
|
|
|
? grpc_static_metadata_hash_values[kidx] |
|
|
|
|
: grpc_slice_hash_refcounted(key); |
|
|
|
|
|
|
|
|
|
uint32_t hash = GRPC_MDSTR_KV_HASH(khash, grpc_slice_hash_refcounted(value)); |
|
|
|
|
return md_create_must_intern<key_definitely_static>(key, value, hash); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
template <bool key_definitely_static> |
|
|
|
|
static grpc_mdelem md_create_must_intern(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value, |
|
|
|
|
uint32_t hash) { |
|
|
|
|
// Here, we know both key and value are both at least interned, and both
|
|
|
|
|
// possibly static. We know that anything inside the shared interned table is
|
|
|
|
|
// also at least interned (and maybe static). Note that equality for a static
|
|
|
|
|
// and interned slice implies that they are both the same exact slice.
|
|
|
|
|
// The same applies to a pair of interned slices, or a pair of static slices.
|
|
|
|
|
// Rather than run the full equality check, we can therefore just do a pointer
|
|
|
|
|
// comparison of the refcounts.
|
|
|
|
|
InternedMetadata* md; |
|
|
|
|
mdtab_shard* shard = &g_shards[SHARD_IDX(hash)]; |
|
|
|
|
size_t idx; |
|
|
|
|
@ -348,7 +446,8 @@ grpc_mdelem grpc_mdelem_create( |
|
|
|
|
idx = TABLE_IDX(hash, shard->capacity); |
|
|
|
|
/* search for an existing pair */ |
|
|
|
|
for (md = shard->elems[idx].next; md; md = md->bucket_next()) { |
|
|
|
|
if (grpc_slice_eq(key, md->key()) && grpc_slice_eq(value, md->value())) { |
|
|
|
|
if (grpc_slice_static_interned_equal(key, md->key()) && |
|
|
|
|
grpc_slice_static_interned_equal(value, md->value())) { |
|
|
|
|
md->RefWithShardLocked(shard); |
|
|
|
|
gpr_mu_unlock(&shard->mu); |
|
|
|
|
return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED); |
|
|
|
|
@ -356,8 +455,12 @@ grpc_mdelem grpc_mdelem_create( |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* not found: create a new pair */ |
|
|
|
|
md = grpc_core::New<InternedMetadata>(key, value, hash, |
|
|
|
|
shard->elems[idx].next); |
|
|
|
|
md = key_definitely_static |
|
|
|
|
? grpc_core::New<InternedMetadata>( |
|
|
|
|
key, value, hash, shard->elems[idx].next, |
|
|
|
|
static_cast<const InternedMetadata::NoRefKey*>(nullptr)) |
|
|
|
|
: grpc_core::New<InternedMetadata>(key, value, hash, |
|
|
|
|
shard->elems[idx].next); |
|
|
|
|
shard->elems[idx].next = md; |
|
|
|
|
shard->count++; |
|
|
|
|
|
|
|
|
|
@ -370,9 +473,68 @@ grpc_mdelem grpc_mdelem_create( |
|
|
|
|
return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_create( |
|
|
|
|
const grpc_slice& key, const grpc_slice& value, |
|
|
|
|
grpc_mdelem_data* compatible_external_backing_store) { |
|
|
|
|
return md_create<false>(key, value, compatible_external_backing_store); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_create( |
|
|
|
|
const grpc_core::StaticMetadataSlice& key, const grpc_slice& value, |
|
|
|
|
grpc_mdelem_data* compatible_external_backing_store) { |
|
|
|
|
return md_create<true>(key, value, compatible_external_backing_store); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/* Create grpc_mdelem from provided slices. We specify via template parameter
|
|
|
|
|
whether we know that the input key is static or not. If it is, we short |
|
|
|
|
circuit various comparisons and a no-op unref. */ |
|
|
|
|
template <bool key_definitely_static> |
|
|
|
|
static grpc_mdelem md_from_slices(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value) { |
|
|
|
|
// Ensure key is, in fact, static if we claimed it was.
|
|
|
|
|
GPR_DEBUG_ASSERT(!key_definitely_static || |
|
|
|
|
GRPC_IS_STATIC_METADATA_STRING(key)); |
|
|
|
|
grpc_mdelem out = md_create<key_definitely_static>(key, value, nullptr); |
|
|
|
|
if (!key_definitely_static) { |
|
|
|
|
grpc_slice_unref_internal(key); |
|
|
|
|
} |
|
|
|
|
grpc_slice_unref_internal(value); |
|
|
|
|
return out; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key, |
|
|
|
|
const grpc_slice& value) { |
|
|
|
|
grpc_mdelem out = grpc_mdelem_create(key, value, nullptr); |
|
|
|
|
return md_from_slices</*key_definitely_static=*/false>(key, value); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_from_slices(const grpc_core::StaticMetadataSlice& key, |
|
|
|
|
const grpc_slice& value) { |
|
|
|
|
return md_from_slices</*key_definitely_static=*/true>(key, value); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_from_slices( |
|
|
|
|
const grpc_core::StaticMetadataSlice& key, |
|
|
|
|
const grpc_core::StaticMetadataSlice& value) { |
|
|
|
|
grpc_mdelem out = md_create_maybe_static<true, true>(key, value); |
|
|
|
|
return out; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_from_slices( |
|
|
|
|
const grpc_core::StaticMetadataSlice& key, |
|
|
|
|
const grpc_core::ManagedMemorySlice& value) { |
|
|
|
|
// TODO(arjunroy): We can save the unref if md_create_maybe_static ended up
|
|
|
|
|
// creating a new interned metadata. But otherwise - we need this here.
|
|
|
|
|
grpc_mdelem out = md_create_maybe_static<true>(key, value); |
|
|
|
|
grpc_slice_unref_internal(value); |
|
|
|
|
return out; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
grpc_mdelem grpc_mdelem_from_slices( |
|
|
|
|
const grpc_core::ManagedMemorySlice& key, |
|
|
|
|
const grpc_core::ManagedMemorySlice& value) { |
|
|
|
|
grpc_mdelem out = md_create_maybe_static<false>(key, value); |
|
|
|
|
// TODO(arjunroy): We can save the unref if md_create_maybe_static ended up
|
|
|
|
|
// creating a new interned metadata. But otherwise - we need this here.
|
|
|
|
|
grpc_slice_unref_internal(key); |
|
|
|
|
grpc_slice_unref_internal(value); |
|
|
|
|
return out; |
|
|
|
|
|
Arjun Roy
6 years ago
committed by
GitHub