diff --git a/BUILD b/BUILD index 9e49b42304..e7c89c5635 100644 --- a/BUILD +++ b/BUILD @@ -86,6 +86,7 @@ cc_library( "upb/table.int.h", "upb/upb.c", "upb/upb.int.h", + "third_party/wyhash/wyhash.h", ], hdrs = [ "upb/decode.h", diff --git a/tests/benchmark.cc b/tests/benchmark.cc index 15544afe1b..1a1d9b55aa 100644 --- a/tests/benchmark.cc +++ b/tests/benchmark.cc @@ -4,6 +4,7 @@ #include "google/protobuf/descriptor.upb.h" #include "google/protobuf/descriptor.upbdefs.h" #include "google/protobuf/descriptor.pb.h" +#include "upb/def.hpp" upb_strview descriptor = google_protobuf_descriptor_proto_upbdefinit.descriptor; namespace protobuf = ::google::protobuf; @@ -11,6 +12,34 @@ namespace protobuf = ::google::protobuf; /* A buffer big enough to parse descriptor.proto without going to heap. */ char buf[65535]; +static void BM_LoadDescriptor(benchmark::State& state) { + FILE *f = fopen("/tmp/ads-descriptor.pb", "rb"); + fseek(f, 0, SEEK_END); + size_t size = ftell(f); + fseek(f, 0, SEEK_SET); + std::string str(size, '\0'); + fread(&str[0], 1, size, f); + fclose(f); + fprintf(stderr, "size: %zu\n", str.size()); + for (auto _ : state) { + upb::SymbolTable symtab; + upb::Arena arena; + google_protobuf_FileDescriptorSet* set = + google_protobuf_FileDescriptorSet_parse(str.data(), str.size(), + arena.ptr()); + const google_protobuf_FileDescriptorProto*const * files = + google_protobuf_FileDescriptorSet_file(set, &size); + for (size_t i = 0; i < size; i++) { + upb::FileDefPtr file_def = symtab.AddFile(files[i], NULL); + if (!file_def) { + printf("Failed to add file.\n"); + exit(1); + } + } + } +} +BENCHMARK(BM_LoadDescriptor); + static void BM_ArenaOneAlloc(benchmark::State& state) { for (auto _ : state) { upb_arena* arena = upb_arena_new(); diff --git a/tools/amalgamate.py b/tools/amalgamate.py index b244eff8e8..950e9aa9ac 100755 --- a/tools/amalgamate.py +++ b/tools/amalgamate.py @@ -52,7 +52,7 @@ class Amalgamator: include = parse_include(line) if not include: return False - if not (include.startswith("upb") or include.startswith("google")): + if not (include.startswith("upb") or include.startswith("google") or include.startswith("third_party")): return False if include.endswith("hpp"): # Skip, we don't support the amalgamation from C++. diff --git a/upb/def.c b/upb/def.c index 497f402b99..603deb2013 100644 --- a/upb/def.c +++ b/upb/def.c @@ -1309,7 +1309,7 @@ static bool create_oneofdef( CHK_OOM(upb_strtable_insert3(&m->ntof, name.data, name.size, v, ctx->alloc)); CHK_OOM(upb_inttable_init2(&o->itof, UPB_CTYPE_CONSTPTR, ctx->alloc)); - CHK_OOM(upb_strtable_init2(&o->ntof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + CHK_OOM(upb_strtable_init2(&o->ntof, UPB_CTYPE_CONSTPTR, 4, ctx->alloc)); return true; } @@ -1633,14 +1633,13 @@ static bool create_enumdef( e->full_name = makefullname(ctx, prefix, name); CHK_OOM(symtab_add(ctx, e->full_name, pack_def(e, UPB_DEFTYPE_ENUM))); - CHK_OOM(upb_strtable_init2(&e->ntoi, UPB_CTYPE_INT32, ctx->alloc)); + values = google_protobuf_EnumDescriptorProto_value(enum_proto, &n); + CHK_OOM(upb_strtable_init2(&e->ntoi, UPB_CTYPE_INT32, n, ctx->alloc)); CHK_OOM(upb_inttable_init2(&e->iton, UPB_CTYPE_CSTR, ctx->alloc)); e->file = ctx->file; e->defaultval = 0; - values = google_protobuf_EnumDescriptorProto_value(enum_proto, &n); - if (n == 0) { upb_status_seterrf(ctx->status, "enums must contain at least one value (%s)", @@ -1690,7 +1689,7 @@ static bool create_msgdef(symtab_addctx *ctx, const char *prefix, const google_protobuf_FieldDescriptorProto *const *fields; const google_protobuf_EnumDescriptorProto *const *enums; const google_protobuf_DescriptorProto *const *msgs; - size_t i, n; + size_t i, n_oneof, n_field, n; upb_strview name; name = google_protobuf_DescriptorProto_name(msg_proto); @@ -1700,8 +1699,12 @@ static bool create_msgdef(symtab_addctx *ctx, const char *prefix, m->full_name = makefullname(ctx, prefix, name); CHK_OOM(symtab_add(ctx, m->full_name, pack_def(m, UPB_DEFTYPE_MSG))); + oneofs = google_protobuf_DescriptorProto_oneof_decl(msg_proto, &n_oneof); + fields = google_protobuf_DescriptorProto_field(msg_proto, &n_field); + CHK_OOM(upb_inttable_init2(&m->itof, UPB_CTYPE_CONSTPTR, ctx->alloc)); - CHK_OOM(upb_strtable_init2(&m->ntof, UPB_CTYPE_CONSTPTR, ctx->alloc)); + CHK_OOM(upb_strtable_init2(&m->ntof, UPB_CTYPE_CONSTPTR, n_oneof + n_field, + ctx->alloc)); m->file = ctx->file; m->map_entry = false; @@ -1720,17 +1723,15 @@ static bool create_msgdef(symtab_addctx *ctx, const char *prefix, m->layout = upb_malloc(ctx->alloc, sizeof(*m->layout)); } - oneofs = google_protobuf_DescriptorProto_oneof_decl(msg_proto, &n); m->oneof_count = 0; - m->oneofs = upb_malloc(ctx->alloc, sizeof(*m->oneofs) * n); - for (i = 0; i < n; i++) { + m->oneofs = upb_malloc(ctx->alloc, sizeof(*m->oneofs) * n_oneof); + for (i = 0; i < n_oneof; i++) { CHK(create_oneofdef(ctx, m, oneofs[i])); } - fields = google_protobuf_DescriptorProto_field(msg_proto, &n); m->field_count = 0; - m->fields = upb_malloc(ctx->alloc, sizeof(*m->fields) * n); - for (i = 0; i < n; i++) { + m->fields = upb_malloc(ctx->alloc, sizeof(*m->fields) * n_field); + for (i = 0; i < n_field; i++) { CHK(create_fielddef(ctx, m->full_name, m, fields[i])); } @@ -2085,8 +2086,8 @@ upb_symtab *upb_symtab_new(void) { s->arena = upb_arena_new(); alloc = upb_arena_alloc(s->arena); - if (!upb_strtable_init2(&s->syms, UPB_CTYPE_CONSTPTR, alloc) || - !upb_strtable_init2(&s->files, UPB_CTYPE_CONSTPTR, alloc)) { + if (!upb_strtable_init2(&s->syms, UPB_CTYPE_CONSTPTR, 32, alloc) || + !upb_strtable_init2(&s->files, UPB_CTYPE_CONSTPTR, 4, alloc)) { upb_arena_free(s->arena); upb_gfree(s); s = NULL; @@ -2148,7 +2149,7 @@ static const upb_filedef *_upb_symtab_addfile( ctx.layouts = layouts; ctx.status = status; - ok = file && upb_strtable_init2(&addtab, UPB_CTYPE_CONSTPTR, ctx.tmp) && + ok = file && upb_strtable_init2(&addtab, UPB_CTYPE_CONSTPTR, 8, ctx.tmp) && build_filedef(&ctx, file, file_proto) && upb_symtab_addtotabs(s, &ctx); upb_arena_free(tmparena); diff --git a/upb/json/parser.rl b/upb/json/parser.rl index d7dcc54a8b..47ba15ec2e 100644 --- a/upb/json/parser.rl +++ b/upb/json/parser.rl @@ -2869,7 +2869,7 @@ static upb_json_parsermethod *parsermethod_new(upb_json_codecache *c, upb_byteshandler_setstring(&m->input_handler_, parse, m); upb_byteshandler_setendstr(&m->input_handler_, end, m); - upb_strtable_init2(&m->name_table, UPB_CTYPE_CONSTPTR, alloc); + upb_strtable_init2(&m->name_table, UPB_CTYPE_CONSTPTR, 4, alloc); /* Build name_table */ diff --git a/upb/msg.c b/upb/msg.c index 5f6d1ce170..baafe59293 100644 --- a/upb/msg.c +++ b/upb/msg.c @@ -130,7 +130,7 @@ upb_map *_upb_map_new(upb_arena *a, size_t key_size, size_t value_size) { return NULL; } - upb_strtable_init2(&map->table, UPB_CTYPE_INT32, upb_arena_alloc(a)); + upb_strtable_init2(&map->table, UPB_CTYPE_INT32, 4, upb_arena_alloc(a)); map->key_size = key_size; map->val_size = value_size; diff --git a/upb/table.c b/upb/table.c index 34a20530d8..9af5163061 100644 --- a/upb/table.c +++ b/upb/table.c @@ -4,10 +4,12 @@ ** Implementation is heavily inspired by Lua's ltable.c. */ -#include "upb/table.int.h" - #include +#include "third_party/wyhash/wyhash.h" +#include "upb/table.int.h" + +/* Must be last. */ #include "upb/port_def.inc" #define UPB_MAXARRSIZE 16 /* 64k. */ @@ -87,11 +89,7 @@ static upb_tabent *mutable_entries(upb_table *t) { } static bool isfull(upb_table *t) { - if (upb_table_size(t) == 0) { - return true; - } else { - return ((double)(t->count + 1) / upb_table_size(t)) > MAX_LOAD; - } + return t->count == t->max_count; } static bool init(upb_table *t, uint8_t size_lg2, upb_alloc *a) { @@ -100,6 +98,7 @@ static bool init(upb_table *t, uint8_t size_lg2, upb_alloc *a) { t->count = 0; t->size_lg2 = size_lg2; t->mask = upb_table_size(t) ? upb_table_size(t) - 1 : 0; + t->max_count = upb_table_size(t) * MAX_LOAD; bytes = upb_table_size(t) * sizeof(upb_tabent); if (bytes > 0) { t->entries = upb_malloc(a, bytes); @@ -115,9 +114,17 @@ static void uninit(upb_table *t, upb_alloc *a) { upb_free(a, mutable_entries(t)); } -static upb_tabent *emptyent(upb_table *t) { - upb_tabent *e = mutable_entries(t) + upb_table_size(t); - while (1) { if (upb_tabent_isempty(--e)) return e; UPB_ASSERT(e > t->entries); } +static upb_tabent *emptyent(upb_table *t, upb_tabent *e) { + upb_tabent *begin = mutable_entries(t); + upb_tabent *end = begin + upb_table_size(t); + for (e = e + 1; e < end; e++) { + if (upb_tabent_isempty(e)) return e; + } + for (e = begin; e < end; e++) { + if (upb_tabent_isempty(e)) return e; + } + UPB_ASSERT(false); + return NULL; } static upb_tabent *getentry_mutable(upb_table *t, uint32_t hash) { @@ -173,7 +180,7 @@ static void insert(upb_table *t, lookupkey_t key, upb_tabkey tabkey, our_e->next = NULL; } else { /* Collision. */ - upb_tabent *new_e = emptyent(t); + upb_tabent *new_e = emptyent(t, mainpos_e); /* Head of collider's chain. */ upb_tabent *chain = getentry_mutable(t, hashfunc(mainpos_e->key)); if (chain == mainpos_e) { @@ -268,10 +275,14 @@ static upb_tabkey strcopy(lookupkey_t k2, upb_alloc *a) { return (uintptr_t)str; } +static uint32_t table_hash(const char *p, size_t n) { + return wyhash(p, n, 0, _wyp); +} + static uint32_t strhash(upb_tabkey key) { uint32_t len; char *str = upb_tabstr(key, &len); - return upb_murmur_hash2(str, len, 0); + return table_hash(str, len); } static bool streql(upb_tabkey k1, lookupkey_t k2) { @@ -280,9 +291,15 @@ static bool streql(upb_tabkey k1, lookupkey_t k2) { return len == k2.str.len && (len == 0 || memcmp(str, k2.str.str, len) == 0); } -bool upb_strtable_init2(upb_strtable *t, upb_ctype_t ctype, upb_alloc *a) { +bool upb_strtable_init2(upb_strtable *t, upb_ctype_t ctype, + size_t expected_size, upb_alloc *a) { UPB_UNUSED(ctype); /* TODO(haberman): rm */ - return init(&t->t, 2, a); + size_t need_entries = (expected_size + 1) / MAX_LOAD; + int size_lg2 = 2; + while (1 << size_lg2 < need_entries) { + size_lg2++; + } + return init(&t->t, size_lg2, a); } void upb_strtable_clear(upb_strtable *t) { @@ -333,20 +350,20 @@ bool upb_strtable_insert3(upb_strtable *t, const char *k, size_t len, tabkey = strcopy(key, a); if (tabkey == 0) return false; - hash = upb_murmur_hash2(key.str.str, key.str.len, 0); + hash = table_hash(key.str.str, key.str.len); insert(&t->t, key, tabkey, v, hash, &strhash, &streql); return true; } bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len, upb_value *v) { - uint32_t hash = upb_murmur_hash2(key, len, 0); + uint32_t hash = table_hash(key, len); return lookup(&t->t, strkey2(key, len), v, hash, &streql); } bool upb_strtable_remove3(upb_strtable *t, const char *key, size_t len, upb_value *val, upb_alloc *alloc) { - uint32_t hash = upb_murmur_hash2(key, len, 0); + uint32_t hash = table_hash(key, len); upb_tabkey tabkey; if (rm(&t->t, strkey2(key, len), val, &tabkey, hash, &streql)) { if (alloc) { @@ -699,182 +716,3 @@ bool upb_inttable_iter_isequal(const upb_inttable_iter *i1, return i1->t == i2->t && i1->index == i2->index && i1->array_part == i2->array_part; } - -#if defined(UPB_UNALIGNED_READS_OK) || defined(__s390x__) -/* ----------------------------------------------------------------------------- - * MurmurHash2, by Austin Appleby (released as public domain). - * Reformatted and C99-ified by Joshua Haberman. - * Note - This code makes a few assumptions about how your machine behaves - - * 1. We can read a 4-byte value from any address without crashing - * 2. sizeof(int) == 4 (in upb this limitation is removed by using uint32_t - * And it has a few limitations - - * 1. It will not work incrementally. - * 2. It will not produce the same results on little-endian and big-endian - * machines. */ -uint32_t upb_murmur_hash2(const void *key, size_t len, uint32_t seed) { - /* 'm' and 'r' are mixing constants generated offline. - * They're not really 'magic', they just happen to work well. */ - const uint32_t m = 0x5bd1e995; - const int32_t r = 24; - - /* Initialize the hash to a 'random' value */ - uint32_t h = seed ^ len; - - /* Mix 4 bytes at a time into the hash */ - const uint8_t * data = (const uint8_t *)key; - while(len >= 4) { - uint32_t k; - memcpy(&k, data, sizeof(k)); - - k *= m; - k ^= k >> r; - k *= m; - - h *= m; - h ^= k; - - data += 4; - len -= 4; - } - - /* Handle the last few bytes of the input array */ - switch(len) { - case 3: h ^= data[2] << 16; - case 2: h ^= data[1] << 8; - case 1: h ^= data[0]; h *= m; - }; - - /* Do a few final mixes of the hash to ensure the last few - * bytes are well-incorporated. */ - h ^= h >> 13; - h *= m; - h ^= h >> 15; - - return h; -} - -#else /* !UPB_UNALIGNED_READS_OK */ - -/* ----------------------------------------------------------------------------- - * MurmurHashAligned2, by Austin Appleby - * Same algorithm as MurmurHash2, but only does aligned reads - should be safer - * on certain platforms. - * Performance will be lower than MurmurHash2 */ - -#define MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; } - -uint32_t upb_murmur_hash2(const void * key, size_t len, uint32_t seed) { - const uint32_t m = 0x5bd1e995; - const int32_t r = 24; - const uint8_t * data = (const uint8_t *)key; - uint32_t h = (uint32_t)(seed ^ len); - uint8_t align = (uintptr_t)data & 3; - - if(align && (len >= 4)) { - /* Pre-load the temp registers */ - uint32_t t = 0, d = 0; - int32_t sl; - int32_t sr; - - switch(align) { - case 1: t |= data[2] << 16; /* fallthrough */ - case 2: t |= data[1] << 8; /* fallthrough */ - case 3: t |= data[0]; - } - - t <<= (8 * align); - - data += 4-align; - len -= 4-align; - - sl = 8 * (4-align); - sr = 8 * align; - - /* Mix */ - - while(len >= 4) { - uint32_t k; - - d = *(uint32_t *)data; - t = (t >> sr) | (d << sl); - - k = t; - - MIX(h,k,m); - - t = d; - - data += 4; - len -= 4; - } - - /* Handle leftover data in temp registers */ - - d = 0; - - if(len >= align) { - uint32_t k; - - switch(align) { - case 3: d |= data[2] << 16; /* fallthrough */ - case 2: d |= data[1] << 8; /* fallthrough */ - case 1: d |= data[0]; /* fallthrough */ - } - - k = (t >> sr) | (d << sl); - MIX(h,k,m); - - data += align; - len -= align; - - /* ---------- - * Handle tail bytes */ - - switch(len) { - case 3: h ^= data[2] << 16; /* fallthrough */ - case 2: h ^= data[1] << 8; /* fallthrough */ - case 1: h ^= data[0]; h *= m; /* fallthrough */ - }; - } else { - switch(len) { - case 3: d |= data[2] << 16; /* fallthrough */ - case 2: d |= data[1] << 8; /* fallthrough */ - case 1: d |= data[0]; /* fallthrough */ - case 0: h ^= (t >> sr) | (d << sl); h *= m; - } - } - - h ^= h >> 13; - h *= m; - h ^= h >> 15; - - return h; - } else { - while(len >= 4) { - uint32_t k = *(uint32_t *)data; - - MIX(h,k,m); - - data += 4; - len -= 4; - } - - /* ---------- - * Handle tail bytes */ - - switch(len) { - case 3: h ^= data[2] << 16; /* fallthrough */ - case 2: h ^= data[1] << 8; /* fallthrough */ - case 1: h ^= data[0]; h *= m; - }; - - h ^= h >> 13; - h *= m; - h ^= h >> 15; - - return h; - } -} -#undef MIX - -#endif /* UPB_UNALIGNED_READS_OK */ diff --git a/upb/table.int.h b/upb/table.int.h index 600637eef2..1e6c232cf8 100644 --- a/upb/table.int.h +++ b/upb/table.int.h @@ -171,7 +171,8 @@ typedef struct _upb_tabent { typedef struct { size_t count; /* Number of entries in the hash part. */ - size_t mask; /* Mask to turn hash value -> bucket. */ + uint32_t mask; /* Mask to turn hash value -> bucket. */ + uint32_t max_count; /* Max count before we hit our load limit. */ uint8_t size_lg2; /* Size of the hashtable part is 2^size_lg2 entries. */ /* Hash table entries. @@ -230,7 +231,8 @@ UPB_INLINE bool upb_arrhas(upb_tabval key) { /* Initialize and uninitialize a table, respectively. If memory allocation * failed, false is returned that the table is uninitialized. */ bool upb_inttable_init2(upb_inttable *table, upb_ctype_t ctype, upb_alloc *a); -bool upb_strtable_init2(upb_strtable *table, upb_ctype_t ctype, upb_alloc *a); +bool upb_strtable_init2(upb_strtable *table, upb_ctype_t ctype, + size_t expected_size, upb_alloc *a); void upb_inttable_uninit2(upb_inttable *table, upb_alloc *a); void upb_strtable_uninit2(upb_strtable *table, upb_alloc *a); @@ -239,7 +241,7 @@ UPB_INLINE bool upb_inttable_init(upb_inttable *table, upb_ctype_t ctype) { } UPB_INLINE bool upb_strtable_init(upb_strtable *table, upb_ctype_t ctype) { - return upb_strtable_init2(table, ctype, &upb_alloc_global); + return upb_strtable_init2(table, ctype, 4, &upb_alloc_global); } UPB_INLINE void upb_inttable_uninit(upb_inttable *table) {