diff --git a/include/grpc/slice.h b/include/grpc/slice.h index b5894a103c1..53a650f3995 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -127,6 +127,8 @@ GPRAPI grpc_slice gpr_empty_slice(void); GPRAPI int grpc_slice_cmp(grpc_slice a, grpc_slice b); GPRAPI int grpc_slice_str_cmp(grpc_slice a, const char *b); +GPRAPI uint32_t grpc_slice_hash(grpc_slice s); + #ifdef __cplusplus } #endif diff --git a/src/core/lib/slice/slice_traits.h b/src/core/lib/slice/slice_traits.h new file mode 100644 index 00000000000..facbd0dd812 --- /dev/null +++ b/src/core/lib/slice/slice_traits.h @@ -0,0 +1,44 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef SLICE_TRAITS_H +#define SLICE_TRAITS_H + +#include +#include + +bool grpc_slice_is_legal_header(grpc_slice s); +bool grpc_slice_is_legal_nonbin_header(grpc_slice s); +bool grpc_slice_is_bin_suffixed(grpc_slice s); + +#endif diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c index 9b5d8099c71..c0743e124a5 100644 --- a/src/core/lib/transport/metadata.c +++ b/src/core/lib/transport/metadata.c @@ -52,8 +52,6 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/static_metadata.h" -grpc_slice (*grpc_chttp2_base64_encode_and_huffman_compress)(grpc_slice input); - /* There are two kinds of mdelem and mdstr instances. * Static instances are declared in static_metadata.{h,c} and * are initialized by grpc_mdctx_global_init(). @@ -63,9 +61,6 @@ grpc_slice (*grpc_chttp2_base64_encode_and_huffman_compress)(grpc_slice input); * used to determine which kind of element a pointer refers to. */ -#define INITIAL_STRTAB_CAPACITY 4 -#define INITIAL_MDTAB_CAPACITY 4 - #ifdef GRPC_METADATA_REFCOUNT_DEBUG #define DEBUG_ARGS , const char *file, int line #define FWD_DEBUG_ARGS , file, line @@ -76,37 +71,20 @@ grpc_slice (*grpc_chttp2_base64_encode_and_huffman_compress)(grpc_slice input); #define REF_MD_LOCKED(shard, s) ref_md_locked((shard), (s)) #endif -#define TABLE_IDX(hash, log2_shards, capacity) \ - (((hash) >> (log2_shards)) % (capacity)) -#define SHARD_IDX(hash, log2_shards) ((hash) & ((1 << (log2_shards)) - 1)) - -typedef void (*destroy_user_data_func)(void *user_data); - -#define SIZE_IN_DECODER_TABLE_NOT_SET -1 -/* Shadow structure for grpc_mdstr for non-static values */ -typedef struct internal_string { - /* must be byte compatible with grpc_mdstr */ - grpc_slice slice; - uint32_t hash; +#define INITIAL_SHARD_CAPACITY 8 +#define LOG2_SHARD_COUNT 4 +#define SHARD_COUNT ((size_t)(1 << LOG2_SHARD_COUNT)) - /* private only data */ - gpr_atm refcnt; +#define TABLE_IDX(hash, capacity) (((hash) >> (LOG2_SHARD_COUNT)) % (capacity)) +#define SHARD_IDX(hash) ((hash) & ((1 << (LOG2_SHARD_COUNT)) - 1)) - uint8_t has_base64_and_huffman_encoded; - grpc_slice_refcount refcount; - - grpc_slice base64_and_huffman; - - gpr_atm size_in_decoder_table; - - struct internal_string *bucket_next; -} internal_string; +typedef void (*destroy_user_data_func)(void *user_data); /* Shadow structure for grpc_mdelem for non-static elements */ typedef struct internal_metadata { /* must be byte compatible with grpc_mdelem */ - internal_string *key; - internal_string *value; + grpc_slice key; + grpc_slice value; /* private only data */ gpr_atm refcnt; @@ -118,13 +96,6 @@ typedef struct internal_metadata { struct internal_metadata *bucket_next; } internal_metadata; -typedef struct strtab_shard { - gpr_mu mu; - internal_string **strs; - size_t count; - size_t capacity; -} strtab_shard; - typedef struct mdtab_shard { gpr_mu mu; internal_metadata **elems; @@ -136,23 +107,16 @@ typedef struct mdtab_shard { gpr_atm free_estimate; } mdtab_shard; -#define LOG2_STRTAB_SHARD_COUNT 5 -#define LOG2_MDTAB_SHARD_COUNT 4 -#define STRTAB_SHARD_COUNT ((size_t)(1 << LOG2_STRTAB_SHARD_COUNT)) -#define MDTAB_SHARD_COUNT ((size_t)(1 << LOG2_MDTAB_SHARD_COUNT)) - /* hash seed: decided at initialization time */ static uint32_t g_hash_seed; static int g_forced_hash_seed = 0; /* linearly probed hash tables for static element lookup */ -static grpc_mdstr *g_static_strtab[GRPC_STATIC_MDSTR_COUNT * 2]; static grpc_mdelem *g_static_mdtab[GRPC_STATIC_MDELEM_COUNT * 2]; static size_t g_static_strtab_maxprobe; static size_t g_static_mdtab_maxprobe; -static strtab_shard g_strtab_shard[STRTAB_SHARD_COUNT]; -static mdtab_shard g_mdtab_shard[MDTAB_SHARD_COUNT]; +static mdtab_shard g_shards[SHARD_COUNT]; static void gc_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard); @@ -162,12 +126,13 @@ void grpc_test_only_set_metadata_hash_seed(uint32_t seed) { } void grpc_mdctx_global_init(void) { - size_t i, j; + size_t i; if (!g_forced_hash_seed) { g_hash_seed = (uint32_t)gpr_now(GPR_CLOCK_REALTIME).tv_nsec; } g_static_strtab_maxprobe = 0; g_static_mdtab_maxprobe = 0; +#if 0 /* build static tables */ memset(g_static_mdtab, 0, sizeof(g_static_mdtab)); memset(g_static_strtab, 0, sizeof(g_static_strtab)); @@ -208,21 +173,14 @@ void grpc_mdctx_global_init(void) { g_static_mdtab_maxprobe = j; } } +#endif /* initialize shards */ - for (i = 0; i < STRTAB_SHARD_COUNT; i++) { - strtab_shard *shard = &g_strtab_shard[i]; - gpr_mu_init(&shard->mu); - shard->count = 0; - shard->capacity = INITIAL_STRTAB_CAPACITY; - shard->strs = gpr_malloc(sizeof(*shard->strs) * shard->capacity); - memset(shard->strs, 0, sizeof(*shard->strs) * shard->capacity); - } - for (i = 0; i < MDTAB_SHARD_COUNT; i++) { - mdtab_shard *shard = &g_mdtab_shard[i]; + for (i = 0; i < SHARD_COUNT; i++) { + mdtab_shard *shard = &g_shards[i]; gpr_mu_init(&shard->mu); shard->count = 0; gpr_atm_no_barrier_store(&shard->free_estimate, 0); - shard->capacity = INITIAL_MDTAB_CAPACITY; + shard->capacity = INITIAL_SHARD_CAPACITY; shard->elems = gpr_malloc(sizeof(*shard->elems) * shard->capacity); memset(shard->elems, 0, sizeof(*shard->elems) * shard->capacity); } @@ -230,8 +188,8 @@ void grpc_mdctx_global_init(void) { void grpc_mdctx_global_shutdown(grpc_exec_ctx *exec_ctx) { size_t i; - for (i = 0; i < MDTAB_SHARD_COUNT; i++) { - mdtab_shard *shard = &g_mdtab_shard[i]; + for (i = 0; i < SHARD_COUNT; i++) { + mdtab_shard *shard = &g_shards[i]; gpr_mu_destroy(&shard->mu); gc_mdtab(exec_ctx, shard); /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */ @@ -244,30 +202,6 @@ void grpc_mdctx_global_shutdown(grpc_exec_ctx *exec_ctx) { } gpr_free(shard->elems); } - for (i = 0; i < STRTAB_SHARD_COUNT; i++) { - strtab_shard *shard = &g_strtab_shard[i]; - gpr_mu_destroy(&shard->mu); - /* TODO(ctiller): GPR_ASSERT(shard->count == 0); */ - if (shard->count != 0) { - gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked", - shard->count); - for (size_t j = 0; j < shard->capacity; j++) { - for (internal_string *s = shard->strs[j]; s; s = s->bucket_next) { - gpr_log(GPR_DEBUG, "LEAKED: %s", - grpc_mdstr_as_c_string((grpc_mdstr *)s)); - } - } - if (grpc_iomgr_abort_on_leaks()) { - abort(); - } - } - gpr_free(shard->strs); - } -} - -static int is_mdstr_static(grpc_mdstr *s) { - return s >= &grpc_static_mdstr_table[0] && - s < &grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT]; } static int is_mdelem_static(grpc_mdelem *e) { @@ -290,162 +224,6 @@ static void ref_md_locked(mdtab_shard *shard, } } -static void grow_strtab(strtab_shard *shard) { - size_t capacity = shard->capacity * 2; - size_t i; - internal_string **strtab; - internal_string *s, *next; - - GPR_TIMER_BEGIN("grow_strtab", 0); - - strtab = gpr_malloc(sizeof(internal_string *) * capacity); - memset(strtab, 0, sizeof(internal_string *) * capacity); - - for (i = 0; i < shard->capacity; i++) { - for (s = shard->strs[i]; s; s = next) { - size_t idx = TABLE_IDX(s->hash, LOG2_STRTAB_SHARD_COUNT, capacity); - next = s->bucket_next; - s->bucket_next = strtab[idx]; - strtab[idx] = s; - } - } - - gpr_free(shard->strs); - shard->strs = strtab; - shard->capacity = capacity; - - GPR_TIMER_END("grow_strtab", 0); -} - -static void internal_destroy_string(grpc_exec_ctx *exec_ctx, - strtab_shard *shard, internal_string *is) { - internal_string **prev_next; - internal_string *cur; - GPR_TIMER_BEGIN("internal_destroy_string", 0); - if (is->has_base64_and_huffman_encoded) { - grpc_slice_unref_internal(exec_ctx, is->base64_and_huffman); - } - for (prev_next = &shard->strs[TABLE_IDX(is->hash, LOG2_STRTAB_SHARD_COUNT, - shard->capacity)], - cur = *prev_next; - cur != is; prev_next = &cur->bucket_next, cur = cur->bucket_next) - ; - *prev_next = cur->bucket_next; - shard->count--; - gpr_free(is); - GPR_TIMER_END("internal_destroy_string", 0); -} - -static void slice_ref(void *p) { - internal_string *is = - (internal_string *)((char *)p - offsetof(internal_string, refcount)); - GRPC_MDSTR_REF((grpc_mdstr *)(is)); -} - -static void slice_unref(grpc_exec_ctx *exec_ctx, void *p) { - internal_string *is = - (internal_string *)((char *)p - offsetof(internal_string, refcount)); - GRPC_MDSTR_UNREF(exec_ctx, (grpc_mdstr *)(is)); -} - -grpc_mdstr *grpc_mdstr_from_string(const char *str) { - return grpc_mdstr_from_buffer((const uint8_t *)str, strlen(str)); -} - -grpc_mdstr *grpc_mdstr_from_slice(grpc_exec_ctx *exec_ctx, grpc_slice slice) { - grpc_mdstr *result = grpc_mdstr_from_buffer(GRPC_SLICE_START_PTR(slice), - GRPC_SLICE_LENGTH(slice)); - grpc_slice_unref_internal(exec_ctx, slice); - return result; -} - -grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) { - uint32_t hash = gpr_murmur_hash3(buf, length, g_hash_seed); - internal_string *s; - strtab_shard *shard = - &g_strtab_shard[SHARD_IDX(hash, LOG2_STRTAB_SHARD_COUNT)]; - size_t i; - size_t idx; - - GPR_TIMER_BEGIN("grpc_mdstr_from_buffer", 0); - - /* search for a static string */ - for (i = 0; i <= g_static_strtab_maxprobe; i++) { - grpc_mdstr *ss; - idx = (hash + i) % GPR_ARRAY_SIZE(g_static_strtab); - ss = g_static_strtab[idx]; - if (ss == NULL) break; - if (ss->hash == hash && GRPC_SLICE_LENGTH(ss->slice) == length && - (length == 0 || - 0 == memcmp(buf, GRPC_SLICE_START_PTR(ss->slice), length))) { - GPR_TIMER_END("grpc_mdstr_from_buffer", 0); - return ss; - } - } - - gpr_mu_lock(&shard->mu); - - /* search for an existing string */ - idx = TABLE_IDX(hash, LOG2_STRTAB_SHARD_COUNT, shard->capacity); - for (s = shard->strs[idx]; s; s = s->bucket_next) { - if (s->hash == hash && GRPC_SLICE_LENGTH(s->slice) == length && - 0 == memcmp(buf, GRPC_SLICE_START_PTR(s->slice), length)) { - if (gpr_atm_full_fetch_add(&s->refcnt, 1) == 0) { - /* If we get here, we've added a ref to something that was about to - * die - drop it immediately. - * The *only* possible path here (given the shard mutex) should be to - * drop from one ref back to zero - assert that with a CAS */ - GPR_ASSERT(gpr_atm_rel_cas(&s->refcnt, 1, 0)); - /* and treat this as if we were never here... sshhh */ - } else { - gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdstr_from_buffer", 0); - return (grpc_mdstr *)s; - } - } - } - - /* not found: create a new string */ - if (length + 1 < GRPC_SLICE_INLINED_SIZE) { - /* string data goes directly into the slice */ - s = gpr_malloc(sizeof(internal_string)); - gpr_atm_rel_store(&s->refcnt, 1); - s->slice.refcount = NULL; - memcpy(s->slice.data.inlined.bytes, buf, length); - s->slice.data.inlined.bytes[length] = 0; - s->slice.data.inlined.length = (uint8_t)length; - } else { - /* string data goes after the internal_string header, and we +1 for null - terminator */ - s = gpr_malloc(sizeof(internal_string) + length + 1); - gpr_atm_rel_store(&s->refcnt, 1); - s->refcount.ref = slice_ref; - s->refcount.unref = slice_unref; - s->slice.refcount = &s->refcount; - s->slice.data.refcounted.bytes = (uint8_t *)(s + 1); - s->slice.data.refcounted.length = length; - memcpy(s->slice.data.refcounted.bytes, buf, length); - /* add a null terminator for cheap c string conversion when desired */ - s->slice.data.refcounted.bytes[length] = 0; - } - s->has_base64_and_huffman_encoded = 0; - s->hash = hash; - s->size_in_decoder_table = SIZE_IN_DECODER_TABLE_NOT_SET; - s->bucket_next = shard->strs[idx]; - shard->strs[idx] = s; - - shard->count++; - - if (shard->count > shard->capacity * 2) { - grow_strtab(shard); - } - - gpr_mu_unlock(&shard->mu); - GPR_TIMER_END("grpc_mdstr_from_buffer", 0); - - return (grpc_mdstr *)s; -} - static void gc_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard) { size_t i; internal_metadata **prev_next; @@ -459,8 +237,8 @@ static void gc_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard) { void *user_data = (void *)gpr_atm_no_barrier_load(&md->user_data); next = md->bucket_next; if (gpr_atm_acq_load(&md->refcnt) == 0) { - GRPC_MDSTR_UNREF(exec_ctx, (grpc_mdstr *)md->key); - GRPC_MDSTR_UNREF(exec_ctx, (grpc_mdstr *)md->value); + grpc_slice_unref_internal(exec_ctx, md->key); + grpc_slice_unref_internal(exec_ctx, md->value); if (md->user_data) { ((destroy_user_data_func)gpr_atm_no_barrier_load( &md->destroy_user_data))(user_data); @@ -493,9 +271,10 @@ static void grow_mdtab(mdtab_shard *shard) { for (i = 0; i < shard->capacity; i++) { for (md = shard->elems[i]; md; md = next) { size_t idx; - hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash); + hash = GRPC_MDSTR_KV_HASH(grpc_slice_hash(md->key), + grpc_slice_hash(md->value)); next = md->bucket_next; - idx = TABLE_IDX(hash, LOG2_MDTAB_SHARD_COUNT, capacity); + idx = TABLE_IDX(hash, capacity); md->bucket_next = mdtab[idx]; mdtab[idx] = md; } @@ -517,26 +296,26 @@ static void rehash_mdtab(grpc_exec_ctx *exec_ctx, mdtab_shard *shard) { } } -grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_exec_ctx *exec_ctx, - grpc_mdstr *mkey, - grpc_mdstr *mvalue) { - internal_string *key = (internal_string *)mkey; - internal_string *value = (internal_string *)mvalue; - uint32_t hash = GRPC_MDSTR_KV_HASH(mkey->hash, mvalue->hash); +grpc_mdelem *grpc_mdelem_from_slices(grpc_exec_ctx *exec_ctx, grpc_slice key, + grpc_slice value) { + uint32_t hash = + GRPC_MDSTR_KV_HASH(grpc_slice_hash(key), grpc_slice_hash(value)); internal_metadata *md; - mdtab_shard *shard = &g_mdtab_shard[SHARD_IDX(hash, LOG2_MDTAB_SHARD_COUNT)]; + mdtab_shard *shard = &g_shards[SHARD_IDX(hash)]; size_t i; size_t idx; GPR_TIMER_BEGIN("grpc_mdelem_from_metadata_strings", 0); - if (is_mdstr_static(mkey) && is_mdstr_static(mvalue)) { + if (grpc_is_static_metadata_string(key) && + grpc_is_static_metadata_string(value)) { for (i = 0; i <= g_static_mdtab_maxprobe; i++) { grpc_mdelem *smd; idx = (hash + i) % GPR_ARRAY_SIZE(g_static_mdtab); smd = g_static_mdtab[idx]; if (smd == NULL) break; - if (smd->key == mkey && smd->value == mvalue) { + if (grpc_slice_cmp(key, smd->key) == 0 && + grpc_slice_cmp(value, smd->value) == 0) { GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); return smd; } @@ -545,14 +324,15 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_exec_ctx *exec_ctx, gpr_mu_lock(&shard->mu); - idx = TABLE_IDX(hash, LOG2_MDTAB_SHARD_COUNT, shard->capacity); + idx = TABLE_IDX(hash, shard->capacity); /* search for an existing pair */ for (md = shard->elems[idx]; md; md = md->bucket_next) { - if (md->key == key && md->value == value) { + if (grpc_slice_cmp(key, md->key) == 0 && + grpc_slice_cmp(value, md->value) == 0) { REF_MD_LOCKED(shard, md); - GRPC_MDSTR_UNREF(exec_ctx, (grpc_mdstr *)key); - GRPC_MDSTR_UNREF(exec_ctx, (grpc_mdstr *)value); gpr_mu_unlock(&shard->mu); + grpc_slice_unref_internal(exec_ctx, key); + grpc_slice_unref_internal(exec_ctx, value); GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); return (grpc_mdelem *)md; } @@ -587,58 +367,19 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_exec_ctx *exec_ctx, return (grpc_mdelem *)md; } -grpc_mdelem *grpc_mdelem_from_strings(grpc_exec_ctx *exec_ctx, const char *key, - const char *value) { - return grpc_mdelem_from_metadata_strings( - exec_ctx, grpc_mdstr_from_string(key), grpc_mdstr_from_string(value)); -} - -grpc_mdelem *grpc_mdelem_from_slices(grpc_exec_ctx *exec_ctx, grpc_slice key, - grpc_slice value) { - return grpc_mdelem_from_metadata_strings( - exec_ctx, grpc_mdstr_from_slice(exec_ctx, key), - grpc_mdstr_from_slice(exec_ctx, value)); -} - -grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_exec_ctx *exec_ctx, - const char *key, - const uint8_t *value, - size_t value_length) { - return grpc_mdelem_from_metadata_strings( - exec_ctx, grpc_mdstr_from_string(key), - grpc_mdstr_from_buffer(value, value_length)); -} - static size_t get_base64_encoded_size(size_t raw_length) { static const uint8_t tail_xtra[3] = {0, 2, 3}; return raw_length / 3 * 4 + tail_xtra[raw_length % 3]; } size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem *elem) { - size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(elem->key->slice); - size_t value_len = GRPC_SLICE_LENGTH(elem->value->slice); - if (is_mdstr_static(elem->value)) { - if (grpc_is_binary_header( - (const char *)GRPC_SLICE_START_PTR(elem->key->slice), - GRPC_SLICE_LENGTH(elem->key->slice))) { - return overhead_and_key + get_base64_encoded_size(value_len); - } else { - return overhead_and_key + value_len; - } + size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(elem->key); + size_t value_len = GRPC_SLICE_LENGTH(elem->value); + if (grpc_is_binary_header((const char *)GRPC_SLICE_START_PTR(elem->key), + GRPC_SLICE_LENGTH(elem->key))) { + return overhead_and_key + get_base64_encoded_size(value_len); } else { - internal_string *is = (internal_string *)elem->value; - gpr_atm current_size = gpr_atm_acq_load(&is->size_in_decoder_table); - if (current_size == SIZE_IN_DECODER_TABLE_NOT_SET) { - if (grpc_is_binary_header( - (const char *)GRPC_SLICE_START_PTR(elem->key->slice), - GRPC_SLICE_LENGTH(elem->key->slice))) { - current_size = (gpr_atm)get_base64_encoded_size(value_len); - } else { - current_size = (gpr_atm)value_len; - } - gpr_atm_rel_store(&is->size_in_decoder_table, current_size); - } - return overhead_and_key + (size_t)current_size; + return overhead_and_key + value_len; } } @@ -674,54 +415,18 @@ void grpc_mdelem_unref(grpc_exec_ctx *exec_ctx, grpc_mdelem *gmd DEBUG_ARGS) { grpc_mdstr_as_c_string((grpc_mdstr *)md->key), grpc_mdstr_as_c_string((grpc_mdstr *)md->value)); #endif - uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash); + uint32_t hash = + GRPC_MDSTR_KV_HASH(grpc_slice_hash(md->key), grpc_slice_hash(md->value)); const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1); GPR_ASSERT(prev_refcount >= 1); if (1 == prev_refcount) { /* once the refcount hits zero, some other thread can come along and free md at any time: it's unsafe from this point on to access it */ - mdtab_shard *shard = - &g_mdtab_shard[SHARD_IDX(hash, LOG2_MDTAB_SHARD_COUNT)]; + mdtab_shard *shard = &g_shards[SHARD_IDX(hash)]; gpr_atm_no_barrier_fetch_add(&shard->free_estimate, 1); } } -const char *grpc_mdstr_as_c_string(const grpc_mdstr *s) { - return (const char *)GRPC_SLICE_START_PTR(s->slice); -} - -size_t grpc_mdstr_length(const grpc_mdstr *s) { return GRPC_MDSTR_LENGTH(s); } - -grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) { - internal_string *s = (internal_string *)gs; - if (is_mdstr_static(gs)) return gs; -#ifdef GRPC_METADATA_REFCOUNT_DEBUG - gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "STR REF:%p:%zu->%zu: '%s'", - (void *)s, gpr_atm_no_barrier_load(&s->refcnt), - gpr_atm_no_barrier_load(&s->refcnt) + 1, grpc_mdstr_as_c_string(gs)); -#endif - GPR_ASSERT(gpr_atm_full_fetch_add(&s->refcnt, 1) > 0); - return gs; -} - -void grpc_mdstr_unref(grpc_exec_ctx *exec_ctx, grpc_mdstr *gs DEBUG_ARGS) { - internal_string *s = (internal_string *)gs; - if (is_mdstr_static(gs)) return; -#ifdef GRPC_METADATA_REFCOUNT_DEBUG - gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "STR UNREF:%p:%zu->%zu: '%s'", - (void *)s, gpr_atm_no_barrier_load(&s->refcnt), - gpr_atm_no_barrier_load(&s->refcnt) - 1, grpc_mdstr_as_c_string(gs)); -#endif - if (1 == gpr_atm_full_fetch_add(&s->refcnt, -1)) { - strtab_shard *shard = - &g_strtab_shard[SHARD_IDX(s->hash, LOG2_STRTAB_SHARD_COUNT)]; - gpr_mu_lock(&shard->mu); - GPR_ASSERT(0 == gpr_atm_no_barrier_load(&s->refcnt)); - internal_destroy_string(exec_ctx, shard, s); - gpr_mu_unlock(&shard->mu); - } -} - void *grpc_mdelem_get_user_data(grpc_mdelem *md, void (*destroy_func)(void *)) { internal_metadata *im = (internal_metadata *)md; void *result; @@ -754,19 +459,3 @@ void grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), gpr_atm_rel_store(&im->destroy_user_data, (gpr_atm)destroy_func); gpr_mu_unlock(&im->mu_user_data); } - -grpc_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *gs) { - internal_string *s = (internal_string *)gs; - grpc_slice slice; - strtab_shard *shard = - &g_strtab_shard[SHARD_IDX(s->hash, LOG2_STRTAB_SHARD_COUNT)]; - gpr_mu_lock(&shard->mu); - if (!s->has_base64_and_huffman_encoded) { - s->base64_and_huffman = - grpc_chttp2_base64_encode_and_huffman_compress(s->slice); - s->has_base64_and_huffman_encoded = 1; - } - slice = s->base64_and_huffman; - gpr_mu_unlock(&shard->mu); - return slice; -} diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 8a64be70250..ede09d7d76c 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -74,51 +74,19 @@ extern "C" { declared here - in which case those functions are effectively no-ops. */ /* Forward declarations */ -typedef struct grpc_mdstr grpc_mdstr; typedef struct grpc_mdelem grpc_mdelem; -/* if changing this, make identical changes in internal_string in metadata.c */ -struct grpc_mdstr { - const grpc_slice slice; - const uint32_t hash; - /* there is a private part to this in metadata.c */ -}; - /* if changing this, make identical changes in internal_metadata in metadata.c */ struct grpc_mdelem { - grpc_mdstr *const key; - grpc_mdstr *const value; + const grpc_slice key; + const grpc_slice value; /* there is a private part to this in metadata.c */ }; -void grpc_test_only_set_metadata_hash_seed(uint32_t seed); - -/* Constructors for grpc_mdstr instances; take a variety of data types that - clients may have handy */ -grpc_mdstr *grpc_mdstr_from_string(const char *str); -/* Unrefs the slice. */ -grpc_mdstr *grpc_mdstr_from_slice(grpc_exec_ctx *exec_ctx, grpc_slice slice); -grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *str, size_t length); - -/* Returns a borrowed slice from the mdstr with its contents base64 encoded - and huffman compressed */ -grpc_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *str); - -/* Constructors for grpc_mdelem instances; take a variety of data types that - clients may have handy */ -grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_exec_ctx *exec_ctx, - grpc_mdstr *key, - grpc_mdstr *value); -grpc_mdelem *grpc_mdelem_from_strings(grpc_exec_ctx *exec_ctx, const char *key, - const char *value); /* Unrefs the slices. */ grpc_mdelem *grpc_mdelem_from_slices(grpc_exec_ctx *exec_ctx, grpc_slice key, grpc_slice value); -grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_exec_ctx *exec_ctx, - const char *key, - const uint8_t *value, - size_t value_length); size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem *elem); @@ -132,52 +100,30 @@ void grpc_mdelem_set_user_data(grpc_mdelem *md, void (*destroy_func)(void *), /* Reference counting */ //#define GRPC_METADATA_REFCOUNT_DEBUG #ifdef GRPC_METADATA_REFCOUNT_DEBUG -#define GRPC_MDSTR_REF(s) grpc_mdstr_ref((s), __FILE__, __LINE__) -#define GRPC_MDSTR_UNREF(exec_ctx, s) \ - grpc_mdstr_unref((exec_ctx), (s), __FILE__, __LINE__) #define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s), __FILE__, __LINE__) #define GRPC_MDELEM_UNREF(exec_ctx, s) \ grpc_mdelem_unref((exec_ctx), (s), __FILE__, __LINE__) -grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *s, const char *file, int line); -void grpc_mdstr_unref(grpc_exec_ctx *exec_ctx, grpc_mdstr *s, const char *file, - int line); grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *md, const char *file, int line); void grpc_mdelem_unref(grpc_exec_ctx *exec_ctx, grpc_mdelem *md, const char *file, int line); #else -#define GRPC_MDSTR_REF(s) grpc_mdstr_ref((s)) -#define GRPC_MDSTR_UNREF(exec_ctx, s) grpc_mdstr_unref((exec_ctx), (s)) #define GRPC_MDELEM_REF(s) grpc_mdelem_ref((s)) #define GRPC_MDELEM_UNREF(exec_ctx, s) grpc_mdelem_unref((exec_ctx), (s)) -grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *s); -void grpc_mdstr_unref(grpc_exec_ctx *exec_ctx, grpc_mdstr *s); grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *md); void grpc_mdelem_unref(grpc_exec_ctx *exec_ctx, grpc_mdelem *md); #endif -/* Recover a char* from a grpc_mdstr. The returned string is null terminated. - Does not promise that the returned string has no embedded nulls however. */ -const char *grpc_mdstr_as_c_string(const grpc_mdstr *s); - #define GRPC_MDSTR_LENGTH(s) (GRPC_SLICE_LENGTH(s->slice)) /* We add 32 bytes of padding as per RFC-7540 section 6.5.2. */ #define GRPC_MDELEM_LENGTH(e) \ (GRPC_MDSTR_LENGTH((e)->key) + GRPC_MDSTR_LENGTH((e)->value) + 32) -int grpc_mdstr_is_legal_header(grpc_mdstr *s); -int grpc_mdstr_is_legal_nonbin_header(grpc_mdstr *s); -int grpc_mdstr_is_bin_suffixed(grpc_mdstr *s); - #define GRPC_MDSTR_KV_HASH(k_hash, v_hash) (GPR_ROTL((k_hash), 2) ^ (v_hash)) void grpc_mdctx_global_init(void); void grpc_mdctx_global_shutdown(grpc_exec_ctx *exec_ctx); -/* Implementation provided by chttp2_transport */ -extern grpc_slice (*grpc_chttp2_base64_encode_and_huffman_compress)( - grpc_slice input); - #ifdef __cplusplus } #endif diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index 8b22592b45d..1d72646a975 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -41,7 +41,336 @@ #include "src/core/lib/transport/static_metadata.h" -grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT]; +static uint8_t g_raw_bytes[] = { + 48, 49, 50, 50, 48, 48, 50, 48, 52, 50, 48, 54, 51, 48, 52, + 52, 48, 48, 52, 48, 52, 53, 48, 48, 97, 99, 99, 101, 112, 116, + 97, 99, 99, 101, 112, 116, 45, 99, 104, 97, 114, 115, 101, 116, 97, + 99, 99, 101, 112, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 97, + 99, 99, 101, 112, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, 97, + 99, 99, 101, 112, 116, 45, 114, 97, 110, 103, 101, 115, 97, 99, 99, + 101, 115, 115, 45, 99, 111, 110, 116, 114, 111, 108, 45, 97, 108, 108, + 111, 119, 45, 111, 114, 105, 103, 105, 110, 97, 103, 101, 97, 108, 108, + 111, 119, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 103, + 114, 112, 99, 58, 97, 117, 116, 104, 111, 114, 105, 116, 121, 97, 117, + 116, 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 99, 97, 99, 104, + 101, 45, 99, 111, 110, 116, 114, 111, 108, 99, 111, 110, 116, 101, 110, + 116, 45, 100, 105, 115, 112, 111, 115, 105, 116, 105, 111, 110, 99, 111, + 110, 116, 101, 110, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 99, + 111, 110, 116, 101, 110, 116, 45, 108, 97, 110, 103, 117, 97, 103, 101, + 99, 111, 110, 116, 101, 110, 116, 45, 108, 101, 110, 103, 116, 104, 99, + 111, 110, 116, 101, 110, 116, 45, 108, 111, 99, 97, 116, 105, 111, 110, + 99, 111, 110, 116, 101, 110, 116, 45, 114, 97, 110, 103, 101, 99, 111, + 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 99, 111, 111, 107, 105, + 101, 100, 97, 116, 101, 100, 101, 102, 108, 97, 116, 101, 100, 101, 102, + 108, 97, 116, 101, 44, 103, 122, 105, 112, 101, 116, 97, 103, 101, 120, + 112, 101, 99, 116, 101, 120, 112, 105, 114, 101, 115, 102, 114, 111, 109, + 71, 69, 84, 103, 114, 112, 99, 103, 114, 112, 99, 45, 97, 99, 99, + 101, 112, 116, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 114, 112, + 99, 45, 101, 110, 99, 111, 100, 105, 110, 103, 103, 114, 112, 99, 45, + 105, 110, 116, 101, 114, 110, 97, 108, 45, 101, 110, 99, 111, 100, 105, + 110, 103, 45, 114, 101, 113, 117, 101, 115, 116, 103, 114, 112, 99, 45, + 109, 101, 115, 115, 97, 103, 101, 103, 114, 112, 99, 45, 112, 97, 121, + 108, 111, 97, 100, 45, 98, 105, 110, 103, 114, 112, 99, 45, 115, 116, + 97, 116, 115, 45, 98, 105, 110, 103, 114, 112, 99, 45, 115, 116, 97, + 116, 117, 115, 103, 114, 112, 99, 45, 116, 105, 109, 101, 111, 117, 116, + 103, 114, 112, 99, 45, 116, 114, 97, 99, 105, 110, 103, 45, 98, 105, + 110, 103, 122, 105, 112, 103, 122, 105, 112, 44, 32, 100, 101, 102, 108, + 97, 116, 101, 104, 111, 115, 116, 104, 116, 116, 112, 104, 116, 116, 112, + 115, 105, 100, 101, 110, 116, 105, 116, 121, 105, 100, 101, 110, 116, 105, + 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, 100, 101, 110, 116, + 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, + 112, 105, 100, 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 105, + 102, 45, 109, 97, 116, 99, 104, 105, 102, 45, 109, 111, 100, 105, 102, + 105, 101, 100, 45, 115, 105, 110, 99, 101, 105, 102, 45, 110, 111, 110, + 101, 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, 97, 110, 103, 101, + 105, 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, + 105, 110, 99, 101, 108, 97, 115, 116, 45, 109, 111, 100, 105, 102, 105, + 101, 100, 108, 98, 45, 99, 111, 115, 116, 45, 98, 105, 110, 108, 98, + 45, 116, 111, 107, 101, 110, 108, 105, 110, 107, 108, 111, 99, 97, 116, + 105, 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, + 58, 109, 101, 116, 104, 111, 100, 58, 112, 97, 116, 104, 80, 79, 83, + 84, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, + 99, 97, 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, + 114, 105, 122, 97, 116, 105, 111, 110, 80, 85, 84, 114, 97, 110, 103, + 101, 114, 101, 102, 101, 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, + 114, 101, 116, 114, 121, 45, 97, 102, 116, 101, 114, 58, 115, 99, 104, + 101, 109, 101, 115, 101, 114, 118, 101, 114, 115, 101, 116, 45, 99, 111, + 111, 107, 105, 101, 47, 47, 105, 110, 100, 101, 120, 46, 104, 116, 109, + 108, 58, 115, 116, 97, 116, 117, 115, 115, 116, 114, 105, 99, 116, 45, + 116, 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, + 105, 116, 121, 116, 101, 116, 114, 97, 105, 108, 101, 114, 115, 116, 114, + 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, 100, 105, 110, 103, + 117, 115, 101, 114, 45, 97, 103, 101, 110, 116, 118, 97, 114, 121, 118, + 105, 97, 119, 119, 119, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, + 97, 116, 101}; + +static void static_ref(void *unused) {} +static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {} +static grpc_slice_refcount g_refcnt = {static_ref, static_unref}; + +bool grpc_is_static_metadata_string(grpc_slice slice) { + return slice.refcount != NULL && slice.refcount->ref == static_ref; +}; + +const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 0, .length = 1}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 1, .length = 1}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 2, .length = 1}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 3, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 6, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 9, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 12, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 15, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 18, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 21, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 24, .length = 6}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 30, .length = 14}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 44, .length = 15}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 59, .length = 15}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 74, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 87, .length = 27}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 114, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 117, .length = 5}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 122, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 138, .length = 10}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 148, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 161, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 174, .length = 19}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 193, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 209, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 225, .length = 14}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 239, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 255, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 268, .length = 12}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 280, .length = 6}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 286, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 290, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 297, .length = 12}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 309, .length = 0}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 309, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 313, .length = 6}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 319, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 326, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 330, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 333, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 337, .length = 20}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 357, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 370, .length = 30}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 400, .length = 12}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 412, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 428, .length = 14}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 442, .length = 11}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 453, .length = 12}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 465, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 481, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 485, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 498, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 502, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 506, .length = 5}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 511, .length = 8}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 519, .length = 16}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 535, .length = 21}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 556, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 569, .length = 8}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 577, .length = 17}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 594, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 607, .length = 8}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 615, .length = 19}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 634, .length = 13}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 647, .length = 11}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 658, .length = 8}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 666, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 670, .length = 8}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 678, .length = 12}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 690, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 697, .length = 5}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 702, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 706, .length = 18}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 724, .length = 19}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 743, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 746, .length = 5}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 751, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 758, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 765, .length = 11}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 776, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 783, .length = 6}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 789, .length = 10}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 799, .length = 1}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 800, .length = 11}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 811, .length = 7}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 818, .length = 25}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 843, .length = 2}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 845, .length = 8}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 853, .length = 17}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 870, .length = 10}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 880, .length = 4}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 884, .length = 3}}, + {.refcount = &g_refcnt, + .data.refcounted = {.bytes = g_raw_bytes + 887, .length = 16}}, +}; + +static const uint8_t g_revmap[] = { + 0, 1, 2, 3, 255, 255, 4, 255, 255, 5, 255, 255, 6, 255, 255, + 7, 255, 255, 8, 255, 255, 9, 255, 255, 10, 255, 255, 255, 255, 255, + 11, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 12, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 13, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 14, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 15, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 16, 255, 255, 17, 255, 255, + 255, 255, 18, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 19, 255, 255, 255, 255, 255, 255, 255, 255, 255, 20, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 21, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 22, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 23, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 24, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 25, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 26, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 27, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 28, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 29, 255, 255, 255, 255, + 255, 30, 255, 255, 255, 31, 255, 255, 255, 255, 255, 255, 32, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 34, 255, 255, 255, 35, 255, + 255, 255, 255, 255, 36, 255, 255, 255, 255, 255, 255, 37, 255, 255, 255, + 38, 255, 255, 39, 255, 255, 255, 40, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 41, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 42, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 43, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 44, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 45, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 46, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 47, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 48, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 49, 255, 255, 255, 50, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 51, 255, 255, 255, 52, 255, 255, 255, 53, 255, 255, 255, + 255, 54, 255, 255, 255, 255, 255, 255, 255, 55, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 56, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 57, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 58, + 255, 255, 255, 255, 255, 255, 255, 59, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 60, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 61, 255, 255, 255, 255, 255, 255, 255, + 62, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 63, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 64, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 65, 255, + 255, 255, 255, 255, 255, 255, 66, 255, 255, 255, 67, 255, 255, 255, 255, + 255, 255, 255, 68, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 69, 255, 255, 255, 255, 255, 255, 70, 255, 255, 255, 255, 71, 255, 255, + 255, 72, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 73, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 74, 255, 255, 75, 255, 255, 255, + 255, 76, 255, 255, 255, 255, 255, 255, 77, 255, 255, 255, 255, 255, 255, + 78, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 79, 255, 255, 255, + 255, 255, 255, 80, 255, 255, 255, 255, 255, 81, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 82, 83, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 84, 255, 255, 255, 255, 255, 255, 85, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 86, 255, 87, 255, 255, 255, 255, 255, 255, 255, 88, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 89, 255, 255, 255, 255, 255, 255, 255, 255, 255, 90, 255, 255, 255, 91, + 255, 255, 92, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255}; + +int grpc_static_metadata_index(grpc_slice slice) { + if (GRPC_SLICE_LENGTH(slice) == 0) return 33; + size_t ofs = (size_t)(GRPC_SLICE_START_PTR(slice) - g_raw_bytes); + if (ofs > sizeof(g_revmap)) return -1; + uint8_t id = g_revmap[ofs]; + return id == 255 ? -1 : id; +}; grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { @@ -50,6 +379,55 @@ uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +#define ELEMS_PHASHLEN 0x40 +#define ELEMS_PHASHNKEYS 81 +#define ELEMS_PHASHRANGE 128 +#define ELEMS_PHASHSALT 0x13c6ef372 + +static const uint8_t elems_tab[] = { + 47, 28, 47, 1, 47, 76, 76, 0, 1, 119, 61, 60, 47, 61, 76, 0, + 0, 32, 61, 76, 0, 0, 1, 0, 0, 0, 0, 0, 0, 101, 0, 0, + 0, 0, 47, 76, 122, 10, 76, 46, 87, 119, 25, 4, 0, 47, 0, 44, + 20, 120, 4, 79, 0, 0, 122, 88, 80, 20, 51, 65, 0, 0, 0, 0, +}; + +static uint32_t elems_phash(uint32_t val) { + val -= 963; + + uint32_t a, b, rsl; + + b = ((val << 19) >> 26); + a = (val & 0x3f); + rsl = (a ^ elems_tab[b]); + return rsl; +} + +static const uint16_t elem_keys[] = { + 3844, 1521, 2544, 7194, 7815, 7816, 7817, 7818, 7819, 7820, 7821, 6357, + 6822, 5706, 2358, 3381, 1428, 6488, 3862, 7386, 2622, 6078, 7101, 1166, + 3195, 3867, 2730, 1335, 6491, 2079, 8496, 5427, 7399, 7400, 1893, 8403, + 3751, 3752, 1149, 8310, 3288, 6729, 7473, 2265, 2451, 6455, 5799, 963, + 5985, 7008, 1056, 4278, 4279, 4280, 3769, 2637, 1242, 6592, 6593, 3774, + 3775, 3776, 3777, 1986, 4776, 5520, 6264, 3474, 7566, 7938, 8217, 1614, + 2823, 1800, 8085, 8589, 7287, 5892, 2172, 6171, 5613, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0}; +static const uint8_t elem_idxs[] = { + 33, 7, 17, 60, 67, 68, 69, 70, 71, 72, 73, 50, 57, 43, 15, 24, 6, + 52, 34, 62, 18, 47, 59, 3, 22, 35, 20, 5, 53, 12, 79, 40, 63, 64, + 10, 78, 26, 27, 2, 77, 23, 56, 65, 14, 16, 51, 44, 1, 46, 58, 0, + 36, 37, 38, 28, 19, 4, 54, 55, 29, 30, 31, 32, 11, 39, 41, 49, 25, + 66, 74, 76, 8, 21, 9, 75, 80, 61, 45, 13, 48, 42}; + +grpc_mdelem *grpc_static_mdelem_for_static_strings(int a, int b) { + if (a == -1 || b == -1) return NULL; + uint32_t k = (uint32_t)(a * 93 + b); + uint32_t h = elems_phash(k); + return elem_keys[h] == k ? &grpc_static_mdelem_table[elem_idxs[h]] : NULL; +} + const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] = {11, 33, 10, 33, 12, 33, 12, 50, 13, 33, 14, 33, 15, 33, 16, 33, 17, 33, 19, 33, 20, 33, 21, 33, 22, 33, 23, 33, 24, 33, 25, 33, 26, 33, 27, 33, @@ -61,100 +439,5 @@ const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2] = 79, 52, 79, 53, 80, 33, 81, 33, 84, 3, 84, 4, 84, 5, 84, 6, 84, 7, 84, 8, 84, 9, 85, 33, 86, 87, 88, 33, 89, 33, 90, 33, 91, 33, 92, 33}; -const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = { - "0", - "1", - "2", - "200", - "204", - "206", - "304", - "400", - "404", - "500", - "accept", - "accept-charset", - "accept-encoding", - "accept-language", - "accept-ranges", - "access-control-allow-origin", - "age", - "allow", - "application/grpc", - ":authority", - "authorization", - "cache-control", - "content-disposition", - "content-encoding", - "content-language", - "content-length", - "content-location", - "content-range", - "content-type", - "cookie", - "date", - "deflate", - "deflate,gzip", - "", - "etag", - "expect", - "expires", - "from", - "GET", - "grpc", - "grpc-accept-encoding", - "grpc-encoding", - "grpc-internal-encoding-request", - "grpc-message", - "grpc-payload-bin", - "grpc-stats-bin", - "grpc-status", - "grpc-timeout", - "grpc-tracing-bin", - "gzip", - "gzip, deflate", - "host", - "http", - "https", - "identity", - "identity,deflate", - "identity,deflate,gzip", - "identity,gzip", - "if-match", - "if-modified-since", - "if-none-match", - "if-range", - "if-unmodified-since", - "last-modified", - "lb-cost-bin", - "lb-token", - "link", - "location", - "max-forwards", - ":method", - ":path", - "POST", - "proxy-authenticate", - "proxy-authorization", - "PUT", - "range", - "referer", - "refresh", - "retry-after", - ":scheme", - "server", - "set-cookie", - "/", - "/index.html", - ":status", - "strict-transport-security", - "te", - "trailers", - "transfer-encoding", - "user-agent", - "vary", - "via", - "www-authenticate"}; - const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 29, 26, 30, 28, 32, 27, 31}; diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index 28ad6f2961a..dccac395cd1 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -45,194 +45,196 @@ #include "src/core/lib/transport/metadata.h" #define GRPC_STATIC_MDSTR_COUNT 93 -extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT]; +extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* "0" */ -#define GRPC_MDSTR_0 (&grpc_static_mdstr_table[0]) +#define GRPC_MDSTR_0 (&grpc_static_slice_table[0]) /* "1" */ -#define GRPC_MDSTR_1 (&grpc_static_mdstr_table[1]) +#define GRPC_MDSTR_1 (&grpc_static_slice_table[1]) /* "2" */ -#define GRPC_MDSTR_2 (&grpc_static_mdstr_table[2]) +#define GRPC_MDSTR_2 (&grpc_static_slice_table[2]) /* "200" */ -#define GRPC_MDSTR_200 (&grpc_static_mdstr_table[3]) +#define GRPC_MDSTR_200 (&grpc_static_slice_table[3]) /* "204" */ -#define GRPC_MDSTR_204 (&grpc_static_mdstr_table[4]) +#define GRPC_MDSTR_204 (&grpc_static_slice_table[4]) /* "206" */ -#define GRPC_MDSTR_206 (&grpc_static_mdstr_table[5]) +#define GRPC_MDSTR_206 (&grpc_static_slice_table[5]) /* "304" */ -#define GRPC_MDSTR_304 (&grpc_static_mdstr_table[6]) +#define GRPC_MDSTR_304 (&grpc_static_slice_table[6]) /* "400" */ -#define GRPC_MDSTR_400 (&grpc_static_mdstr_table[7]) +#define GRPC_MDSTR_400 (&grpc_static_slice_table[7]) /* "404" */ -#define GRPC_MDSTR_404 (&grpc_static_mdstr_table[8]) +#define GRPC_MDSTR_404 (&grpc_static_slice_table[8]) /* "500" */ -#define GRPC_MDSTR_500 (&grpc_static_mdstr_table[9]) +#define GRPC_MDSTR_500 (&grpc_static_slice_table[9]) /* "accept" */ -#define GRPC_MDSTR_ACCEPT (&grpc_static_mdstr_table[10]) +#define GRPC_MDSTR_ACCEPT (&grpc_static_slice_table[10]) /* "accept-charset" */ -#define GRPC_MDSTR_ACCEPT_CHARSET (&grpc_static_mdstr_table[11]) +#define GRPC_MDSTR_ACCEPT_CHARSET (&grpc_static_slice_table[11]) /* "accept-encoding" */ -#define GRPC_MDSTR_ACCEPT_ENCODING (&grpc_static_mdstr_table[12]) +#define GRPC_MDSTR_ACCEPT_ENCODING (&grpc_static_slice_table[12]) /* "accept-language" */ -#define GRPC_MDSTR_ACCEPT_LANGUAGE (&grpc_static_mdstr_table[13]) +#define GRPC_MDSTR_ACCEPT_LANGUAGE (&grpc_static_slice_table[13]) /* "accept-ranges" */ -#define GRPC_MDSTR_ACCEPT_RANGES (&grpc_static_mdstr_table[14]) +#define GRPC_MDSTR_ACCEPT_RANGES (&grpc_static_slice_table[14]) /* "access-control-allow-origin" */ -#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (&grpc_static_mdstr_table[15]) +#define GRPC_MDSTR_ACCESS_CONTROL_ALLOW_ORIGIN (&grpc_static_slice_table[15]) /* "age" */ -#define GRPC_MDSTR_AGE (&grpc_static_mdstr_table[16]) +#define GRPC_MDSTR_AGE (&grpc_static_slice_table[16]) /* "allow" */ -#define GRPC_MDSTR_ALLOW (&grpc_static_mdstr_table[17]) +#define GRPC_MDSTR_ALLOW (&grpc_static_slice_table[17]) /* "application/grpc" */ -#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (&grpc_static_mdstr_table[18]) +#define GRPC_MDSTR_APPLICATION_SLASH_GRPC (&grpc_static_slice_table[18]) /* ":authority" */ -#define GRPC_MDSTR_AUTHORITY (&grpc_static_mdstr_table[19]) +#define GRPC_MDSTR_AUTHORITY (&grpc_static_slice_table[19]) /* "authorization" */ -#define GRPC_MDSTR_AUTHORIZATION (&grpc_static_mdstr_table[20]) +#define GRPC_MDSTR_AUTHORIZATION (&grpc_static_slice_table[20]) /* "cache-control" */ -#define GRPC_MDSTR_CACHE_CONTROL (&grpc_static_mdstr_table[21]) +#define GRPC_MDSTR_CACHE_CONTROL (&grpc_static_slice_table[21]) /* "content-disposition" */ -#define GRPC_MDSTR_CONTENT_DISPOSITION (&grpc_static_mdstr_table[22]) +#define GRPC_MDSTR_CONTENT_DISPOSITION (&grpc_static_slice_table[22]) /* "content-encoding" */ -#define GRPC_MDSTR_CONTENT_ENCODING (&grpc_static_mdstr_table[23]) +#define GRPC_MDSTR_CONTENT_ENCODING (&grpc_static_slice_table[23]) /* "content-language" */ -#define GRPC_MDSTR_CONTENT_LANGUAGE (&grpc_static_mdstr_table[24]) +#define GRPC_MDSTR_CONTENT_LANGUAGE (&grpc_static_slice_table[24]) /* "content-length" */ -#define GRPC_MDSTR_CONTENT_LENGTH (&grpc_static_mdstr_table[25]) +#define GRPC_MDSTR_CONTENT_LENGTH (&grpc_static_slice_table[25]) /* "content-location" */ -#define GRPC_MDSTR_CONTENT_LOCATION (&grpc_static_mdstr_table[26]) +#define GRPC_MDSTR_CONTENT_LOCATION (&grpc_static_slice_table[26]) /* "content-range" */ -#define GRPC_MDSTR_CONTENT_RANGE (&grpc_static_mdstr_table[27]) +#define GRPC_MDSTR_CONTENT_RANGE (&grpc_static_slice_table[27]) /* "content-type" */ -#define GRPC_MDSTR_CONTENT_TYPE (&grpc_static_mdstr_table[28]) +#define GRPC_MDSTR_CONTENT_TYPE (&grpc_static_slice_table[28]) /* "cookie" */ -#define GRPC_MDSTR_COOKIE (&grpc_static_mdstr_table[29]) +#define GRPC_MDSTR_COOKIE (&grpc_static_slice_table[29]) /* "date" */ -#define GRPC_MDSTR_DATE (&grpc_static_mdstr_table[30]) +#define GRPC_MDSTR_DATE (&grpc_static_slice_table[30]) /* "deflate" */ -#define GRPC_MDSTR_DEFLATE (&grpc_static_mdstr_table[31]) +#define GRPC_MDSTR_DEFLATE (&grpc_static_slice_table[31]) /* "deflate,gzip" */ -#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (&grpc_static_mdstr_table[32]) +#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (&grpc_static_slice_table[32]) /* "" */ -#define GRPC_MDSTR_EMPTY (&grpc_static_mdstr_table[33]) +#define GRPC_MDSTR_EMPTY (&grpc_static_slice_table[33]) /* "etag" */ -#define GRPC_MDSTR_ETAG (&grpc_static_mdstr_table[34]) +#define GRPC_MDSTR_ETAG (&grpc_static_slice_table[34]) /* "expect" */ -#define GRPC_MDSTR_EXPECT (&grpc_static_mdstr_table[35]) +#define GRPC_MDSTR_EXPECT (&grpc_static_slice_table[35]) /* "expires" */ -#define GRPC_MDSTR_EXPIRES (&grpc_static_mdstr_table[36]) +#define GRPC_MDSTR_EXPIRES (&grpc_static_slice_table[36]) /* "from" */ -#define GRPC_MDSTR_FROM (&grpc_static_mdstr_table[37]) +#define GRPC_MDSTR_FROM (&grpc_static_slice_table[37]) /* "GET" */ -#define GRPC_MDSTR_GET (&grpc_static_mdstr_table[38]) +#define GRPC_MDSTR_GET (&grpc_static_slice_table[38]) /* "grpc" */ -#define GRPC_MDSTR_GRPC (&grpc_static_mdstr_table[39]) +#define GRPC_MDSTR_GRPC (&grpc_static_slice_table[39]) /* "grpc-accept-encoding" */ -#define GRPC_MDSTR_GRPC_ACCEPT_ENCODING (&grpc_static_mdstr_table[40]) +#define GRPC_MDSTR_GRPC_ACCEPT_ENCODING (&grpc_static_slice_table[40]) /* "grpc-encoding" */ -#define GRPC_MDSTR_GRPC_ENCODING (&grpc_static_mdstr_table[41]) +#define GRPC_MDSTR_GRPC_ENCODING (&grpc_static_slice_table[41]) /* "grpc-internal-encoding-request" */ -#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (&grpc_static_mdstr_table[42]) +#define GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST (&grpc_static_slice_table[42]) /* "grpc-message" */ -#define GRPC_MDSTR_GRPC_MESSAGE (&grpc_static_mdstr_table[43]) +#define GRPC_MDSTR_GRPC_MESSAGE (&grpc_static_slice_table[43]) /* "grpc-payload-bin" */ -#define GRPC_MDSTR_GRPC_PAYLOAD_BIN (&grpc_static_mdstr_table[44]) +#define GRPC_MDSTR_GRPC_PAYLOAD_BIN (&grpc_static_slice_table[44]) /* "grpc-stats-bin" */ -#define GRPC_MDSTR_GRPC_STATS_BIN (&grpc_static_mdstr_table[45]) +#define GRPC_MDSTR_GRPC_STATS_BIN (&grpc_static_slice_table[45]) /* "grpc-status" */ -#define GRPC_MDSTR_GRPC_STATUS (&grpc_static_mdstr_table[46]) +#define GRPC_MDSTR_GRPC_STATUS (&grpc_static_slice_table[46]) /* "grpc-timeout" */ -#define GRPC_MDSTR_GRPC_TIMEOUT (&grpc_static_mdstr_table[47]) +#define GRPC_MDSTR_GRPC_TIMEOUT (&grpc_static_slice_table[47]) /* "grpc-tracing-bin" */ -#define GRPC_MDSTR_GRPC_TRACING_BIN (&grpc_static_mdstr_table[48]) +#define GRPC_MDSTR_GRPC_TRACING_BIN (&grpc_static_slice_table[48]) /* "gzip" */ -#define GRPC_MDSTR_GZIP (&grpc_static_mdstr_table[49]) +#define GRPC_MDSTR_GZIP (&grpc_static_slice_table[49]) /* "gzip, deflate" */ -#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (&grpc_static_mdstr_table[50]) +#define GRPC_MDSTR_GZIP_COMMA_DEFLATE (&grpc_static_slice_table[50]) /* "host" */ -#define GRPC_MDSTR_HOST (&grpc_static_mdstr_table[51]) +#define GRPC_MDSTR_HOST (&grpc_static_slice_table[51]) /* "http" */ -#define GRPC_MDSTR_HTTP (&grpc_static_mdstr_table[52]) +#define GRPC_MDSTR_HTTP (&grpc_static_slice_table[52]) /* "https" */ -#define GRPC_MDSTR_HTTPS (&grpc_static_mdstr_table[53]) +#define GRPC_MDSTR_HTTPS (&grpc_static_slice_table[53]) /* "identity" */ -#define GRPC_MDSTR_IDENTITY (&grpc_static_mdstr_table[54]) +#define GRPC_MDSTR_IDENTITY (&grpc_static_slice_table[54]) /* "identity,deflate" */ -#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (&grpc_static_mdstr_table[55]) +#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (&grpc_static_slice_table[55]) /* "identity,deflate,gzip" */ #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (&grpc_static_mdstr_table[56]) + (&grpc_static_slice_table[56]) /* "identity,gzip" */ -#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (&grpc_static_mdstr_table[57]) +#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (&grpc_static_slice_table[57]) /* "if-match" */ -#define GRPC_MDSTR_IF_MATCH (&grpc_static_mdstr_table[58]) +#define GRPC_MDSTR_IF_MATCH (&grpc_static_slice_table[58]) /* "if-modified-since" */ -#define GRPC_MDSTR_IF_MODIFIED_SINCE (&grpc_static_mdstr_table[59]) +#define GRPC_MDSTR_IF_MODIFIED_SINCE (&grpc_static_slice_table[59]) /* "if-none-match" */ -#define GRPC_MDSTR_IF_NONE_MATCH (&grpc_static_mdstr_table[60]) +#define GRPC_MDSTR_IF_NONE_MATCH (&grpc_static_slice_table[60]) /* "if-range" */ -#define GRPC_MDSTR_IF_RANGE (&grpc_static_mdstr_table[61]) +#define GRPC_MDSTR_IF_RANGE (&grpc_static_slice_table[61]) /* "if-unmodified-since" */ -#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (&grpc_static_mdstr_table[62]) +#define GRPC_MDSTR_IF_UNMODIFIED_SINCE (&grpc_static_slice_table[62]) /* "last-modified" */ -#define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_mdstr_table[63]) +#define GRPC_MDSTR_LAST_MODIFIED (&grpc_static_slice_table[63]) /* "lb-cost-bin" */ -#define GRPC_MDSTR_LB_COST_BIN (&grpc_static_mdstr_table[64]) +#define GRPC_MDSTR_LB_COST_BIN (&grpc_static_slice_table[64]) /* "lb-token" */ -#define GRPC_MDSTR_LB_TOKEN (&grpc_static_mdstr_table[65]) +#define GRPC_MDSTR_LB_TOKEN (&grpc_static_slice_table[65]) /* "link" */ -#define GRPC_MDSTR_LINK (&grpc_static_mdstr_table[66]) +#define GRPC_MDSTR_LINK (&grpc_static_slice_table[66]) /* "location" */ -#define GRPC_MDSTR_LOCATION (&grpc_static_mdstr_table[67]) +#define GRPC_MDSTR_LOCATION (&grpc_static_slice_table[67]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_mdstr_table[68]) +#define GRPC_MDSTR_MAX_FORWARDS (&grpc_static_slice_table[68]) /* ":method" */ -#define GRPC_MDSTR_METHOD (&grpc_static_mdstr_table[69]) +#define GRPC_MDSTR_METHOD (&grpc_static_slice_table[69]) /* ":path" */ -#define GRPC_MDSTR_PATH (&grpc_static_mdstr_table[70]) +#define GRPC_MDSTR_PATH (&grpc_static_slice_table[70]) /* "POST" */ -#define GRPC_MDSTR_POST (&grpc_static_mdstr_table[71]) +#define GRPC_MDSTR_POST (&grpc_static_slice_table[71]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_mdstr_table[72]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (&grpc_static_slice_table[72]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_mdstr_table[73]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (&grpc_static_slice_table[73]) /* "PUT" */ -#define GRPC_MDSTR_PUT (&grpc_static_mdstr_table[74]) +#define GRPC_MDSTR_PUT (&grpc_static_slice_table[74]) /* "range" */ -#define GRPC_MDSTR_RANGE (&grpc_static_mdstr_table[75]) +#define GRPC_MDSTR_RANGE (&grpc_static_slice_table[75]) /* "referer" */ -#define GRPC_MDSTR_REFERER (&grpc_static_mdstr_table[76]) +#define GRPC_MDSTR_REFERER (&grpc_static_slice_table[76]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (&grpc_static_mdstr_table[77]) +#define GRPC_MDSTR_REFRESH (&grpc_static_slice_table[77]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_mdstr_table[78]) +#define GRPC_MDSTR_RETRY_AFTER (&grpc_static_slice_table[78]) /* ":scheme" */ -#define GRPC_MDSTR_SCHEME (&grpc_static_mdstr_table[79]) +#define GRPC_MDSTR_SCHEME (&grpc_static_slice_table[79]) /* "server" */ -#define GRPC_MDSTR_SERVER (&grpc_static_mdstr_table[80]) +#define GRPC_MDSTR_SERVER (&grpc_static_slice_table[80]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (&grpc_static_mdstr_table[81]) +#define GRPC_MDSTR_SET_COOKIE (&grpc_static_slice_table[81]) /* "/" */ -#define GRPC_MDSTR_SLASH (&grpc_static_mdstr_table[82]) +#define GRPC_MDSTR_SLASH (&grpc_static_slice_table[82]) /* "/index.html" */ -#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_mdstr_table[83]) +#define GRPC_MDSTR_SLASH_INDEX_DOT_HTML (&grpc_static_slice_table[83]) /* ":status" */ -#define GRPC_MDSTR_STATUS (&grpc_static_mdstr_table[84]) +#define GRPC_MDSTR_STATUS (&grpc_static_slice_table[84]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_mdstr_table[85]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (&grpc_static_slice_table[85]) /* "te" */ -#define GRPC_MDSTR_TE (&grpc_static_mdstr_table[86]) +#define GRPC_MDSTR_TE (&grpc_static_slice_table[86]) /* "trailers" */ -#define GRPC_MDSTR_TRAILERS (&grpc_static_mdstr_table[87]) +#define GRPC_MDSTR_TRAILERS (&grpc_static_slice_table[87]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_mdstr_table[88]) +#define GRPC_MDSTR_TRANSFER_ENCODING (&grpc_static_slice_table[88]) /* "user-agent" */ -#define GRPC_MDSTR_USER_AGENT (&grpc_static_mdstr_table[89]) +#define GRPC_MDSTR_USER_AGENT (&grpc_static_slice_table[89]) /* "vary" */ -#define GRPC_MDSTR_VARY (&grpc_static_mdstr_table[90]) +#define GRPC_MDSTR_VARY (&grpc_static_slice_table[90]) /* "via" */ -#define GRPC_MDSTR_VIA (&grpc_static_mdstr_table[91]) +#define GRPC_MDSTR_VIA (&grpc_static_slice_table[91]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_mdstr_table[92]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (&grpc_static_slice_table[92]) + +bool grpc_is_static_metadata_string(grpc_slice slice); #define GRPC_STATIC_MDELEM_COUNT 81 extern grpc_mdelem grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; @@ -409,9 +411,9 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "www-authenticate": "" */ #define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (&grpc_static_mdelem_table[80]) +grpc_mdelem *grpc_static_mdelem_for_static_strings(int a, int b); extern const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT * 2]; -extern const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT]; extern const uint8_t grpc_static_accept_encoding_metadata[8]; #define GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS(algs) \ (&grpc_static_mdelem_table[grpc_static_accept_encoding_metadata[(algs)]]) diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index cf3762dbb8d..388bdc611da 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -31,8 +31,11 @@ import hashlib import itertools +import collections import os import sys +import subprocess +import re # configuration: a list of either strings or 2-tuples of strings # a single string represents a static grpc_mdstr @@ -281,12 +284,43 @@ print >>C, '#include "src/core/lib/transport/static_metadata.h"' print >>C print >>H, '#define GRPC_STATIC_MDSTR_COUNT %d' % len(all_strs) -print >>H, 'extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];' +print >>H, 'extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];' for i, elem in enumerate(all_strs): print >>H, '/* "%s" */' % elem - print >>H, '#define %s (&grpc_static_mdstr_table[%d])' % (mangle(elem).upper(), i) + print >>H, '#define %s (&grpc_static_slice_table[%d])' % (mangle(elem).upper(), i) print >>H -print >>C, 'grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];' +print >>H, 'bool grpc_is_static_metadata_string(grpc_slice slice);' +print >>H +print >>C, 'static uint8_t g_raw_bytes[] = {%s};' % (','.join('%d' % ord(c) for c in ''.join(all_strs))) +print >>C +print >>C, 'static void static_ref(void *unused) {}' +print >>C, 'static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {}' +print >>C, 'static grpc_slice_refcount g_refcnt = {static_ref, static_unref};' +print >>C +print >>C, 'bool grpc_is_static_metadata_string(grpc_slice slice) {' +print >>C, ' return slice.refcount != NULL && slice.refcount->ref == static_ref;' +print >>C, '};' +print >>C +print >>C, 'const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = {' +str_ofs = 0 +revmap = {} +zero_length_idx = None +for i, elem in enumerate(all_strs): + print >>C, '{.refcount = &g_refcnt, .data.refcounted = {.bytes = g_raw_bytes+%d, .length=%d}},' % (str_ofs, len(elem)) + revmap[str_ofs] = i + if len(elem) == 0: zero_length_idx = i + str_ofs += len(elem); +print >>C, '};' +print >>C +print >>C, 'static const uint8_t g_revmap[] = {%s};' % ','.join('%d' % (revmap[i] if i in revmap else 255) for i in range(0, str_ofs)) +print >>C +print >>C, 'int grpc_static_metadata_index(grpc_slice slice) {' +print >>C, ' if (GRPC_SLICE_LENGTH(slice) == 0) return %d;' % zero_length_idx +print >>C, ' size_t ofs = (size_t)(GRPC_SLICE_START_PTR(slice) - g_raw_bytes);' +print >>C, ' if (ofs > sizeof(g_revmap)) return -1;' +print >>C, ' uint8_t id = g_revmap[ofs];' +print >>C, ' return id == 255 ? -1 : id;' +print >>C, '};' print >>C print >>D, '# hpack fuzzing dictionary' @@ -319,18 +353,72 @@ def md_idx(m): if m == m2: return i +def perfect_hash(keys, name): + tmp = open('/tmp/keys.txt', 'w') + tmp.write(''.join('%d\n' % (x - min(keys)) for x in keys)) + tmp.close() + cmd = '%s/perfect/run.sh %s -ds' % (os.path.dirname(sys.argv[0]), tmp.name) + subprocess.check_call(cmd, shell=True) + + code = '' + + results = {} + with open('%s/perfect/phash.h' % os.path.dirname(sys.argv[0])) as f: + txt = f.read() + for var in ('PHASHLEN', 'PHASHNKEYS', 'PHASHRANGE', 'PHASHSALT'): + val = re.search(r'#define %s ([0-9a-zA-Z]+)' % var, txt).group(1) + code += '#define %s_%s %s\n' % (name.upper(), var, val) + results[var] = val + code += '\n' + pycode = 'def f(val):\n' + pycode += ' val -= %d\n' % min(keys) + with open('%s/perfect/phash.c' % os.path.dirname(sys.argv[0])) as f: + txt = f.read() + tabdata = re.search(r'ub1 tab\[\] = \{([^}]+)\}', txt, re.MULTILINE).group(1) + code += 'static const uint8_t %s_tab[] = {%s};\n\n' % (name, tabdata) + func_body = re.search(r'ub4 phash\(val\)\nub4 val;\n\{([^}]+)\}', txt, re.MULTILINE).group(1).replace('ub4', 'uint32_t') + code += 'static uint32_t %s_phash(uint32_t val) {\nval -= %d;\n%s}\n' % (name, + min(keys), func_body.replace('tab', '%s_tab' % name)) + pycode += ' tab=(%s)' % tabdata.replace('\n', '') + pycode += '\n'.join(' %s' % s.strip() for s in func_body.splitlines()[2:]) + g = {} + exec pycode in g + pyfunc = g['f'] + + results['code'] = code + results['pyfunc'] = pyfunc + return results + +elem_keys = [str_idx(elem[0]) * len(all_strs) + str_idx(elem[1]) for elem in all_elems] +elem_hash = perfect_hash(elem_keys, "elems") +print >>C, elem_hash['code'] + +keys = [0] * int(elem_hash['PHASHRANGE']) +idxs = [-1] * int(elem_hash['PHASHNKEYS']) +for i, k in enumerate(elem_keys): + h = elem_hash['pyfunc'](k) + assert keys[h] == 0 + keys[h] = k + idxs[h] = i +print >>C, 'static const uint16_t elem_keys[] = {%s};' % ','.join('%d' % k for k in keys) +print >>C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join('%d' % i for i in idxs) +print >>C + +print >>H, 'grpc_mdelem *grpc_static_mdelem_for_static_strings(int a, int b);' +print >>C, 'grpc_mdelem *grpc_static_mdelem_for_static_strings(int a, int b) {' +print >>C, ' if (a == -1 || b == -1) return NULL;' +print >>C, ' uint32_t k = (uint32_t)(a * %d + b);' % len(all_strs) +print >>C, ' uint32_t h = elems_phash(k);' +print >>C, ' return elem_keys[h] == k ? &grpc_static_mdelem_table[elem_idxs[h]] : NULL;' +print >>C, '}' +print >>C + print >>H, 'extern const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT*2];' print >>C, 'const uint8_t grpc_static_metadata_elem_indices[GRPC_STATIC_MDELEM_COUNT*2] = {' print >>C, ','.join('%d' % str_idx(x) for x in itertools.chain.from_iterable([a,b] for a, b in all_elems)) print >>C, '};' print >>C -print >>H, 'extern const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT];' -print >>C, 'const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {' -print >>C, '%s' % ',\n'.join(' "%s"' % s for s in all_strs) -print >>C, '};' -print >>C - print >>H, 'extern const uint8_t grpc_static_accept_encoding_metadata[%d];' % (1 << len(COMPRESSION_ALGORITHMS)) print >>C, 'const uint8_t grpc_static_accept_encoding_metadata[%d] = {' % (1 << len(COMPRESSION_ALGORITHMS)) print >>C, '0,%s' % ','.join('%d' % md_idx(elem) for elem in compression_elems)