@ -68,8 +68,8 @@ void grpc_mdelem_trace_ref(void* md, const grpc_slice& key,
char * key_str = grpc_slice_to_c_string ( key ) ;
char * value_str = grpc_slice_to_c_string ( value ) ;
gpr_log ( file , line , GPR_LOG_SEVERITY_DEBUG ,
" ELM REF:%p:%" PRIdPTR " ->% " PRIdPTR " : '%s' = '%s' " , md , refcnt ,
refcnt + 1 , key_str , value_str ) ;
" mdelem REF:%p:%" PRIdPTR " ->% " PRIdPTR " : '%s' = '%s' " , md ,
refcnt , refcnt + 1 , key_str , value_str ) ;
gpr_free ( key_str ) ;
gpr_free ( value_str ) ;
}
@ -82,7 +82,7 @@ void grpc_mdelem_trace_unref(void* md, const grpc_slice& key,
char * key_str = grpc_slice_to_c_string ( key ) ;
char * value_str = grpc_slice_to_c_string ( value ) ;
gpr_log ( file , line , GPR_LOG_SEVERITY_DEBUG ,
" ELM UNREF:%p:%" PRIdPTR " ->% " PRIdPTR " : '%s' = '%s' " , md ,
" mdelem UNREF:%p:%" PRIdPTR " ->% " PRIdPTR " : '%s' = '%s' " , md ,
refcnt , refcnt - 1 , key_str , value_str ) ;
gpr_free ( key_str ) ;
gpr_free ( value_str ) ;
@ -112,14 +112,33 @@ AllocatedMetadata::AllocatedMetadata(const grpc_slice& key,
: RefcountedMdBase ( grpc_slice_ref_internal ( key ) ,
grpc_slice_ref_internal ( value ) ) {
# ifndef NDEBUG
if ( grpc_trace_metadata . enabled ( ) ) {
char * key_str = grpc_slice_to_c_string ( key ) ;
char * value_str = grpc_slice_to_c_string ( value ) ;
gpr_log ( GPR_DEBUG , " ELM ALLOC:%p:% " PRIdPTR " : '%s' = '%s' " , this ,
RefValue ( ) , key_str , value_str ) ;
gpr_free ( key_str ) ;
gpr_free ( value_str ) ;
}
TraceAtStart ( " ALLOC_MD " ) ;
# endif
}
AllocatedMetadata : : AllocatedMetadata ( const grpc_slice & key ,
const grpc_slice & value , const NoRefKey * )
: RefcountedMdBase ( key , grpc_slice_ref_internal ( value ) ) {
# ifndef NDEBUG
TraceAtStart ( " ALLOC_MD_NOREF_KEY " ) ;
# endif
}
AllocatedMetadata : : AllocatedMetadata (
const grpc_core : : ManagedMemorySlice & key ,
const grpc_core : : UnmanagedMemorySlice & value )
: RefcountedMdBase ( key , value ) {
# ifndef NDEBUG
TraceAtStart ( " ALLOC_MD_NOREF_KEY_VAL " ) ;
# endif
}
AllocatedMetadata : : AllocatedMetadata (
const grpc_core : : ExternallyManagedSlice & key ,
const grpc_core : : UnmanagedMemorySlice & value )
: RefcountedMdBase ( key , value ) {
# ifndef NDEBUG
TraceAtStart ( " ALLOC_MD_NOREF_KEY_VAL " ) ;
# endif
}
@ -134,6 +153,19 @@ AllocatedMetadata::~AllocatedMetadata() {
}
}
# ifndef NDEBUG
void grpc_core : : RefcountedMdBase : : TraceAtStart ( const char * tag ) {
if ( grpc_trace_metadata . enabled ( ) ) {
char * key_str = grpc_slice_to_c_string ( key ( ) ) ;
char * value_str = grpc_slice_to_c_string ( value ( ) ) ;
gpr_log ( GPR_DEBUG , " mdelem %s:%p:% " PRIdPTR " : '%s' = '%s' " , tag , this ,
RefValue ( ) , key_str , value_str ) ;
gpr_free ( key_str ) ;
gpr_free ( value_str ) ;
}
}
# endif
InternedMetadata : : InternedMetadata ( const grpc_slice & key ,
const grpc_slice & value , uint32_t hash ,
InternedMetadata * next )
@ -141,14 +173,16 @@ InternedMetadata::InternedMetadata(const grpc_slice& key,
grpc_slice_ref_internal ( value ) , hash ) ,
link_ ( next ) {
# ifndef NDEBUG
if ( grpc_trace_metadata . enabled ( ) ) {
char * key_str = grpc_slice_to_c_string ( key ) ;
char * value_str = grpc_slice_to_c_string ( value ) ;
gpr_log ( GPR_DEBUG , " ELM NEW:%p:% " PRIdPTR " : '%s' = '%s' " , this ,
RefValue ( ) , key_str , value_str ) ;
gpr_free ( key_str ) ;
gpr_free ( value_str ) ;
}
TraceAtStart ( " INTERNED_MD " ) ;
# endif
}
InternedMetadata : : InternedMetadata ( const grpc_slice & key ,
const grpc_slice & value , uint32_t hash ,
InternedMetadata * next , const NoRefKey * )
: RefcountedMdBase ( key , grpc_slice_ref_internal ( value ) , hash ) , link_ ( next ) {
# ifndef NDEBUG
TraceAtStart ( " INTERNED_MD_NOREF_KEY " ) ;
# endif
}
@ -248,8 +282,8 @@ void InternedMetadata::RefWithShardLocked(mdtab_shard* shard) {
char * value_str = grpc_slice_to_c_string ( value ( ) ) ;
intptr_t value = RefValue ( ) ;
gpr_log ( __FILE__ , __LINE__ , GPR_LOG_SEVERITY_DEBUG ,
" ELM REF:%p:%" PRIdPTR " ->% " PRIdPTR " : '%s' = '%s' " , this , value ,
value + 1 , key_str , value_str ) ;
" mdelem REF:%p:%" PRIdPTR " ->% " PRIdPTR " : '%s' = '%s' " , this ,
value , value + 1 , key_str , value_str ) ;
gpr_free ( key_str ) ;
gpr_free ( value_str ) ;
}
@ -307,36 +341,100 @@ static void rehash_mdtab(mdtab_shard* shard) {
}
}
grpc_mdelem grpc_mdelem_create (
template < bool key_definitely_static , bool value_definitely_static = false >
static grpc_mdelem md_create_maybe_static ( const grpc_slice & key ,
const grpc_slice & value ) ;
template < bool key_definitely_static >
static grpc_mdelem md_create_must_intern ( const grpc_slice & key ,
const grpc_slice & value ,
uint32_t hash ) ;
template < bool key_definitely_static , bool value_definitely_static = false >
static grpc_mdelem md_create (
const grpc_slice & key , const grpc_slice & value ,
grpc_mdelem_data * compatible_external_backing_store ) {
// Ensure slices are, in fact, static if we claimed they were.
GPR_DEBUG_ASSERT ( ! key_definitely_static | |
GRPC_IS_STATIC_METADATA_STRING ( key ) ) ;
GPR_DEBUG_ASSERT ( ! value_definitely_static | |
GRPC_IS_STATIC_METADATA_STRING ( value ) ) ;
const bool key_is_interned =
key_definitely_static | | grpc_slice_is_interned ( key ) ;
const bool value_is_interned =
value_definitely_static | | grpc_slice_is_interned ( value ) ;
// External storage if either slice is not interned and the caller already
// created a backing store. If no backing store, we allocate one.
if ( ! grpc_slice_is_interned ( key ) | | ! grpc_slice_is_interned ( value ) ) {
if ( ! key_is_interned | | ! value_is_interned ) {
if ( compatible_external_backing_store ! = nullptr ) {
// Caller provided backing store.
return GRPC_MAKE_MDELEM ( compatible_external_backing_store ,
GRPC_MDELEM_STORAGE_EXTERNAL ) ;
} else {
// We allocate backing store.
return GRPC_MAKE_MDELEM ( grpc_core : : New < AllocatedMetadata > ( key , value ) ,
GRPC_MDELEM_STORAGE_ALLOCATED ) ;
return key_definitely_static
? GRPC_MAKE_MDELEM (
grpc_core : : New < AllocatedMetadata > (
key , value ,
static_cast < const AllocatedMetadata : : NoRefKey * > (
nullptr ) ) ,
GRPC_MDELEM_STORAGE_ALLOCATED )
: GRPC_MAKE_MDELEM (
grpc_core : : New < AllocatedMetadata > ( key , value ) ,
GRPC_MDELEM_STORAGE_ALLOCATED ) ;
}
}
return md_create_maybe_static < key_definitely_static , value_definitely_static > (
key , value ) ;
}
template < bool key_definitely_static , bool value_definitely_static >
static grpc_mdelem md_create_maybe_static ( const grpc_slice & key ,
const grpc_slice & value ) {
// Ensure slices are, in fact, static if we claimed they were.
GPR_DEBUG_ASSERT ( ! key_definitely_static | |
GRPC_IS_STATIC_METADATA_STRING ( key ) ) ;
GPR_DEBUG_ASSERT ( ! value_definitely_static | |
GRPC_IS_STATIC_METADATA_STRING ( value ) ) ;
GPR_DEBUG_ASSERT ( key . refcount ! = nullptr ) ;
GPR_DEBUG_ASSERT ( value . refcount ! = nullptr ) ;
const bool key_is_static_mdstr =
key_definitely_static | |
key . refcount - > GetType ( ) = = grpc_slice_refcount : : Type : : STATIC ;
const bool value_is_static_mdstr =
value_definitely_static | |
value . refcount - > GetType ( ) = = grpc_slice_refcount : : Type : : STATIC ;
const intptr_t kidx = GRPC_STATIC_METADATA_INDEX ( key ) ;
// Not all static slice input yields a statically stored metadata element.
// It may be worth documenting why.
if ( GRPC_IS_STATIC_METADATA_STRING ( key ) & &
GRPC_IS_STATIC_METADATA_STRING ( value ) ) {
if ( key_is_static_mdstr & & value_is_static_mdstr ) {
grpc_mdelem static_elem = grpc_static_mdelem_for_static_strings (
GRPC_STATIC_METADATA_INDEX ( key ) , GRPC_STATIC_METADATA_INDEX ( value ) ) ;
kidx , GRPC_STATIC_METADATA_INDEX ( value ) ) ;
if ( ! GRPC_MDISNULL ( static_elem ) ) {
return static_elem ;
}
}
uint32_t hash = GRPC_MDSTR_KV_HASH ( grpc_slice_hash_refcounted ( key ) ,
grpc_slice_hash_refcounted ( value ) ) ;
uint32_t khash = key_definitely_static
? grpc_static_metadata_hash_values [ kidx ]
: grpc_slice_hash_refcounted ( key ) ;
uint32_t hash = GRPC_MDSTR_KV_HASH ( khash , grpc_slice_hash_refcounted ( value ) ) ;
return md_create_must_intern < key_definitely_static > ( key , value , hash ) ;
}
template < bool key_definitely_static >
static grpc_mdelem md_create_must_intern ( const grpc_slice & key ,
const grpc_slice & value ,
uint32_t hash ) {
// Here, we know both key and value are both at least interned, and both
// possibly static. We know that anything inside the shared interned table is
// also at least interned (and maybe static). Note that equality for a static
// and interned slice implies that they are both the same exact slice.
// The same applies to a pair of interned slices, or a pair of static slices.
// Rather than run the full equality check, we can therefore just do a pointer
// comparison of the refcounts.
InternedMetadata * md ;
mdtab_shard * shard = & g_shards [ SHARD_IDX ( hash ) ] ;
size_t idx ;
@ -348,7 +446,8 @@ grpc_mdelem grpc_mdelem_create(
idx = TABLE_IDX ( hash , shard - > capacity ) ;
/* search for an existing pair */
for ( md = shard - > elems [ idx ] . next ; md ; md = md - > bucket_next ( ) ) {
if ( grpc_slice_eq ( key , md - > key ( ) ) & & grpc_slice_eq ( value , md - > value ( ) ) ) {
if ( grpc_slice_static_interned_equal ( key , md - > key ( ) ) & &
grpc_slice_static_interned_equal ( value , md - > value ( ) ) ) {
md - > RefWithShardLocked ( shard ) ;
gpr_mu_unlock ( & shard - > mu ) ;
return GRPC_MAKE_MDELEM ( md , GRPC_MDELEM_STORAGE_INTERNED ) ;
@ -356,8 +455,12 @@ grpc_mdelem grpc_mdelem_create(
}
/* not found: create a new pair */
md = grpc_core : : New < InternedMetadata > ( key , value , hash ,
shard - > elems [ idx ] . next ) ;
md = key_definitely_static
? grpc_core : : New < InternedMetadata > (
key , value , hash , shard - > elems [ idx ] . next ,
static_cast < const InternedMetadata : : NoRefKey * > ( nullptr ) )
: grpc_core : : New < InternedMetadata > ( key , value , hash ,
shard - > elems [ idx ] . next ) ;
shard - > elems [ idx ] . next = md ;
shard - > count + + ;
@ -370,9 +473,68 @@ grpc_mdelem grpc_mdelem_create(
return GRPC_MAKE_MDELEM ( md , GRPC_MDELEM_STORAGE_INTERNED ) ;
}
grpc_mdelem grpc_mdelem_create (
const grpc_slice & key , const grpc_slice & value ,
grpc_mdelem_data * compatible_external_backing_store ) {
return md_create < false > ( key , value , compatible_external_backing_store ) ;
}
grpc_mdelem grpc_mdelem_create (
const grpc_core : : StaticMetadataSlice & key , const grpc_slice & value ,
grpc_mdelem_data * compatible_external_backing_store ) {
return md_create < true > ( key , value , compatible_external_backing_store ) ;
}
/* Create grpc_mdelem from provided slices. We specify via template parameter
whether we know that the input key is static or not . If it is , we short
circuit various comparisons and a no - op unref . */
template < bool key_definitely_static >
static grpc_mdelem md_from_slices ( const grpc_slice & key ,
const grpc_slice & value ) {
// Ensure key is, in fact, static if we claimed it was.
GPR_DEBUG_ASSERT ( ! key_definitely_static | |
GRPC_IS_STATIC_METADATA_STRING ( key ) ) ;
grpc_mdelem out = md_create < key_definitely_static > ( key , value , nullptr ) ;
if ( ! key_definitely_static ) {
grpc_slice_unref_internal ( key ) ;
}
grpc_slice_unref_internal ( value ) ;
return out ;
}
grpc_mdelem grpc_mdelem_from_slices ( const grpc_slice & key ,
const grpc_slice & value ) {
grpc_mdelem out = grpc_mdelem_create ( key , value , nullptr ) ;
return md_from_slices < /*key_definitely_static=*/ false > ( key , value ) ;
}
grpc_mdelem grpc_mdelem_from_slices ( const grpc_core : : StaticMetadataSlice & key ,
const grpc_slice & value ) {
return md_from_slices < /*key_definitely_static=*/ true > ( key , value ) ;
}
grpc_mdelem grpc_mdelem_from_slices (
const grpc_core : : StaticMetadataSlice & key ,
const grpc_core : : StaticMetadataSlice & value ) {
grpc_mdelem out = md_create_maybe_static < true , true > ( key , value ) ;
return out ;
}
grpc_mdelem grpc_mdelem_from_slices (
const grpc_core : : StaticMetadataSlice & key ,
const grpc_core : : ManagedMemorySlice & value ) {
// TODO(arjunroy): We can save the unref if md_create_maybe_static ended up
// creating a new interned metadata. But otherwise - we need this here.
grpc_mdelem out = md_create_maybe_static < true > ( key , value ) ;
grpc_slice_unref_internal ( value ) ;
return out ;
}
grpc_mdelem grpc_mdelem_from_slices (
const grpc_core : : ManagedMemorySlice & key ,
const grpc_core : : ManagedMemorySlice & value ) {
grpc_mdelem out = md_create_maybe_static < false > ( key , value ) ;
// TODO(arjunroy): We can save the unref if md_create_maybe_static ended up
// creating a new interned metadata. But otherwise - we need this here.
grpc_slice_unref_internal ( key ) ;
grpc_slice_unref_internal ( value ) ;
return out ;