From 438ecaeb5ae98c8e98d62947d3069121e0bbbf3e Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 3 Oct 2020 19:18:45 -0700 Subject: [PATCH] Give all field parsers a generic table entry. --- CMakeLists.txt | 1 + benchmark.py | 6 +- .../google/protobuf/descriptor.upb.c | 999 ++++++++++++++++++ upb/decode.c | 23 +- upb/decode.h | 19 +- upb/decode_fast.c | 37 +- upb/def.c | 5 + upb/msg.h | 10 + upbc/generator.cc | 8 + 9 files changed, 1071 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dd6454321..f5ce67f2f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ add_library(port upb/port.c) add_library(upb upb/decode.c + upb/decode_fast.c upb/encode.c upb/msg.c upb/msg.h diff --git a/benchmark.py b/benchmark.py index b75dd4d5a8..2e36a2ddb7 100755 --- a/benchmark.py +++ b/benchmark.py @@ -30,12 +30,12 @@ def Run(cmd): def Benchmark(outbase, runs=12): tmpfile = "/tmp/bench-output.json" Run("rm -rf {}".format(tmpfile)) - Run("bazel test :all") - Run("bazel build -c opt :benchmark") + Run("CC=clang bazel test :all") + Run("CC=clang bazel build -c opt :benchmark") Run("./bazel-bin/benchmark --benchmark_out_format=json --benchmark_out={} --benchmark_repetitions={}".format(tmpfile, runs)) - Run("bazel build -c opt --copt=-g :conformance_upb") + Run("CC=clang bazel build -c opt --copt=-g :conformance_upb") Run("cp -f bazel-bin/conformance_upb {}.bin".format(outbase)) with open(tmpfile) as f: diff --git a/generated_for_cmake/google/protobuf/descriptor.upb.c b/generated_for_cmake/google/protobuf/descriptor.upb.c index 44cd3ae2ce..d3a197c849 100644 --- a/generated_for_cmake/google/protobuf/descriptor.upb.c +++ b/generated_for_cmake/google/protobuf/descriptor.upb.c @@ -21,6 +21,43 @@ static const upb_msglayout_field google_protobuf_FileDescriptorSet__fields[1] = }; const upb_msglayout google_protobuf_FileDescriptorSet_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_FileDescriptorSet_submsgs[0], &google_protobuf_FileDescriptorSet__fields[0], UPB_SIZE(4, 8), 1, false, @@ -51,6 +88,43 @@ static const upb_msglayout_field google_protobuf_FileDescriptorProto__fields[12] }; const upb_msglayout google_protobuf_FileDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_FileDescriptorProto_submsgs[0], &google_protobuf_FileDescriptorProto__fields[0], UPB_SIZE(64, 128), 12, false, @@ -80,6 +154,43 @@ static const upb_msglayout_field google_protobuf_DescriptorProto__fields[10] = { }; const upb_msglayout google_protobuf_DescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_DescriptorProto_submsgs[0], &google_protobuf_DescriptorProto__fields[0], UPB_SIZE(48, 96), 10, false, @@ -96,6 +207,43 @@ static const upb_msglayout_field google_protobuf_DescriptorProto_ExtensionRange_ }; const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_DescriptorProto_ExtensionRange_submsgs[0], &google_protobuf_DescriptorProto_ExtensionRange__fields[0], UPB_SIZE(16, 24), 3, false, @@ -107,6 +255,43 @@ static const upb_msglayout_field google_protobuf_DescriptorProto_ReservedRange__ }; const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, NULL, &google_protobuf_DescriptorProto_ReservedRange__fields[0], UPB_SIZE(12, 12), 2, false, @@ -121,6 +306,43 @@ static const upb_msglayout_field google_protobuf_ExtensionRangeOptions__fields[1 }; const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_ExtensionRangeOptions_submsgs[0], &google_protobuf_ExtensionRangeOptions__fields[0], UPB_SIZE(4, 8), 1, false, @@ -145,6 +367,43 @@ static const upb_msglayout_field google_protobuf_FieldDescriptorProto__fields[11 }; const upb_msglayout google_protobuf_FieldDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_FieldDescriptorProto_submsgs[0], &google_protobuf_FieldDescriptorProto__fields[0], UPB_SIZE(80, 128), 11, false, @@ -160,6 +419,43 @@ static const upb_msglayout_field google_protobuf_OneofDescriptorProto__fields[2] }; const upb_msglayout google_protobuf_OneofDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_OneofDescriptorProto_submsgs[0], &google_protobuf_OneofDescriptorProto__fields[0], UPB_SIZE(16, 32), 2, false, @@ -180,6 +476,43 @@ static const upb_msglayout_field google_protobuf_EnumDescriptorProto__fields[5] }; const upb_msglayout google_protobuf_EnumDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_EnumDescriptorProto_submsgs[0], &google_protobuf_EnumDescriptorProto__fields[0], UPB_SIZE(32, 64), 5, false, @@ -191,6 +524,43 @@ static const upb_msglayout_field google_protobuf_EnumDescriptorProto_EnumReserve }; const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, NULL, &google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[0], UPB_SIZE(12, 12), 2, false, @@ -207,6 +577,43 @@ static const upb_msglayout_field google_protobuf_EnumValueDescriptorProto__field }; const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_EnumValueDescriptorProto_submsgs[0], &google_protobuf_EnumValueDescriptorProto__fields[0], UPB_SIZE(24, 32), 3, false, @@ -224,6 +631,43 @@ static const upb_msglayout_field google_protobuf_ServiceDescriptorProto__fields[ }; const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_ServiceDescriptorProto_submsgs[0], &google_protobuf_ServiceDescriptorProto__fields[0], UPB_SIZE(24, 48), 3, false, @@ -243,6 +687,43 @@ static const upb_msglayout_field google_protobuf_MethodDescriptorProto__fields[6 }; const upb_msglayout google_protobuf_MethodDescriptorProto_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_MethodDescriptorProto_submsgs[0], &google_protobuf_MethodDescriptorProto__fields[0], UPB_SIZE(32, 64), 6, false, @@ -277,6 +758,43 @@ static const upb_msglayout_field google_protobuf_FileOptions__fields[21] = { }; const upb_msglayout google_protobuf_FileOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_FileOptions_submsgs[0], &google_protobuf_FileOptions__fields[0], UPB_SIZE(112, 208), 21, false, @@ -295,6 +813,43 @@ static const upb_msglayout_field google_protobuf_MessageOptions__fields[5] = { }; const upb_msglayout google_protobuf_MessageOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_MessageOptions_submsgs[0], &google_protobuf_MessageOptions__fields[0], UPB_SIZE(12, 16), 5, false, @@ -315,6 +870,43 @@ static const upb_msglayout_field google_protobuf_FieldOptions__fields[7] = { }; const upb_msglayout google_protobuf_FieldOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_FieldOptions_submsgs[0], &google_protobuf_FieldOptions__fields[0], UPB_SIZE(32, 40), 7, false, @@ -329,6 +921,43 @@ static const upb_msglayout_field google_protobuf_OneofOptions__fields[1] = { }; const upb_msglayout google_protobuf_OneofOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_OneofOptions_submsgs[0], &google_protobuf_OneofOptions__fields[0], UPB_SIZE(4, 8), 1, false, @@ -345,6 +974,43 @@ static const upb_msglayout_field google_protobuf_EnumOptions__fields[3] = { }; const upb_msglayout google_protobuf_EnumOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_EnumOptions_submsgs[0], &google_protobuf_EnumOptions__fields[0], UPB_SIZE(8, 16), 3, false, @@ -360,6 +1026,43 @@ static const upb_msglayout_field google_protobuf_EnumValueOptions__fields[2] = { }; const upb_msglayout google_protobuf_EnumValueOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_EnumValueOptions_submsgs[0], &google_protobuf_EnumValueOptions__fields[0], UPB_SIZE(8, 16), 2, false, @@ -375,6 +1078,43 @@ static const upb_msglayout_field google_protobuf_ServiceOptions__fields[2] = { }; const upb_msglayout google_protobuf_ServiceOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_ServiceOptions_submsgs[0], &google_protobuf_ServiceOptions__fields[0], UPB_SIZE(8, 16), 2, false, @@ -391,6 +1131,43 @@ static const upb_msglayout_field google_protobuf_MethodOptions__fields[3] = { }; const upb_msglayout google_protobuf_MethodOptions_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_MethodOptions_submsgs[0], &google_protobuf_MethodOptions__fields[0], UPB_SIZE(24, 32), 3, false, @@ -411,6 +1188,43 @@ static const upb_msglayout_field google_protobuf_UninterpretedOption__fields[7] }; const upb_msglayout google_protobuf_UninterpretedOption_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_UninterpretedOption_submsgs[0], &google_protobuf_UninterpretedOption__fields[0], UPB_SIZE(64, 96), 7, false, @@ -422,6 +1236,43 @@ static const upb_msglayout_field google_protobuf_UninterpretedOption_NamePart__f }; const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, NULL, &google_protobuf_UninterpretedOption_NamePart__fields[0], UPB_SIZE(16, 32), 2, false, @@ -436,6 +1287,43 @@ static const upb_msglayout_field google_protobuf_SourceCodeInfo__fields[1] = { }; const upb_msglayout google_protobuf_SourceCodeInfo_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_SourceCodeInfo_submsgs[0], &google_protobuf_SourceCodeInfo__fields[0], UPB_SIZE(4, 8), 1, false, @@ -450,6 +1338,43 @@ static const upb_msglayout_field google_protobuf_SourceCodeInfo_Location__fields }; const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, NULL, &google_protobuf_SourceCodeInfo_Location__fields[0], UPB_SIZE(32, 64), 5, false, @@ -464,6 +1389,43 @@ static const upb_msglayout_field google_protobuf_GeneratedCodeInfo__fields[1] = }; const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, &google_protobuf_GeneratedCodeInfo_submsgs[0], &google_protobuf_GeneratedCodeInfo__fields[0], UPB_SIZE(4, 8), 1, false, @@ -477,6 +1439,43 @@ static const upb_msglayout_field google_protobuf_GeneratedCodeInfo_Annotation__f }; const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit = { + { + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + &fastdecode_generic, + }, + { + 0 + }, NULL, &google_protobuf_GeneratedCodeInfo_Annotation__fields[0], UPB_SIZE(24, 48), 4, false, diff --git a/upb/decode.c b/upb/decode.c index f3b151aadd..2130f1e583 100644 --- a/upb/decode.c +++ b/upb/decode.c @@ -275,6 +275,7 @@ static upb_msg *decode_newsubmsg(upb_decstate *d, const upb_msglayout *layout, return _upb_msg_new(subl, d->arena); } +UPB_FORCEINLINE static void decode_tosubmsg(upb_decstate *d, upb_msg *submsg, const upb_msglayout *layout, const upb_msglayout_field *field, upb_strview val) { @@ -288,6 +289,7 @@ static void decode_tosubmsg(upb_decstate *d, upb_msg *submsg, d->depth++; } +UPB_FORCEINLINE static const char *decode_group(upb_decstate *d, const char *ptr, upb_msg *submsg, const upb_msglayout *subl, uint32_t number) { @@ -299,6 +301,7 @@ static const char *decode_group(upb_decstate *d, const char *ptr, return ptr; } +UPB_FORCEINLINE static const char *decode_togroup(upb_decstate *d, const char *ptr, upb_msg *submsg, const upb_msglayout *layout, const upb_msglayout_field *field) { @@ -306,6 +309,7 @@ static const char *decode_togroup(upb_decstate *d, const char *ptr, return decode_group(d, ptr, submsg, subl, field->number); } +UPB_FORCEINLINE static const char *decode_toarray(upb_decstate *d, const char *ptr, upb_msg *msg, const upb_msglayout *layout, const upb_msglayout_field *field, wireval val, @@ -398,6 +402,7 @@ static const char *decode_toarray(upb_decstate *d, const char *ptr, } } +UPB_FORCEINLINE static void decode_tomap(upb_decstate *d, upb_msg *msg, const upb_msglayout *layout, const upb_msglayout_field *field, wireval val) { @@ -434,6 +439,7 @@ static void decode_tomap(upb_decstate *d, upb_msg *msg, _upb_map_set(map, &ent.k, map->key_size, &ent.v, map->val_size, d->arena); } +UPB_FORCEINLINE static const char *decode_tomsg(upb_decstate *d, const char *ptr, upb_msg *msg, const upb_msglayout *layout, const upb_msglayout_field *field, wireval val, @@ -491,8 +497,9 @@ static const char *decode_tomsg(upb_decstate *d, const char *ptr, upb_msg *msg, return ptr; } -const char *decode_field(upb_decstate *d, const char *ptr, upb_msg *msg, - const upb_msglayout *layout) { +UPB_FORCEINLINE +static const char *decode_field(upb_decstate *d, const char *ptr, upb_msg *msg, + const upb_msglayout *layout) { uint32_t tag; const upb_msglayout_field *field; int field_number; @@ -587,10 +594,19 @@ const char *decode_field(upb_decstate *d, const char *ptr, upb_msg *msg, return ptr; } +const char *fastdecode_generic(upb_decstate *d, const char *ptr, upb_msg *msg, + const upb_msglayout *table, uint64_t hasbits, + uint64_t data) { + *(uint32_t*)msg |= hasbits; /* Sync hasbits. */ + (void)data; + return decode_field(d, ptr, msg, table); +} + static const char *decode_msg(upb_decstate *d, const char *ptr, upb_msg *msg, const upb_msglayout *layout) { while (ptr < d->limit) { - ptr = decode_field(d, ptr, msg, layout); + ptr = fastdecode_dispatch(d, ptr, msg, layout, 0); + if (ptr < d->limit) ptr = decode_field(d, ptr, msg, layout); } if (ptr != d->limit) decode_err(d); @@ -601,6 +617,7 @@ bool upb_decode(const char *buf, size_t size, void *msg, const upb_msglayout *l, upb_arena *arena) { upb_decstate state; state.limit = buf + size; + state.fastlimit = buf + size - 16; state.arena = arena; state.depth = 64; state.end_group = 0; diff --git a/upb/decode.h b/upb/decode.h index c6387190d2..6fc4a2da91 100644 --- a/upb/decode.h +++ b/upb/decode.h @@ -16,19 +16,6 @@ extern "C" { bool upb_decode(const char *buf, size_t size, upb_msg *msg, const upb_msglayout *l, upb_arena *arena); -struct upb_fasttable; -struct upb_decstate; - -typedef const char *_upb_field_parser(struct upb_decstate *d, const char *ptr, - upb_msg *msg, struct upb_fasttable *table, - uint64_t hasbits, uint64_t data); - -typedef struct upb_fasttable { - _upb_field_parser *field_parser[16]; - uint64_t field_data[16]; - _upb_field_parser *fallback; -} upb_fasttable; - /* Internal only: data pertaining to the parse. */ typedef struct upb_decstate { char *arena_ptr, *arena_end; @@ -43,6 +30,12 @@ typedef struct upb_decstate { jmp_buf err; } upb_decstate; +const char *fastdecode_dispatch(upb_decstate *d, const char *ptr, upb_msg *msg, + const upb_msglayout *table, uint64_t hasbits); +const char *fastdecode_generic(upb_decstate *d, const char *ptr, upb_msg *msg, + const upb_msglayout *table, uint64_t hasbits, + uint64_t data); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/upb/decode_fast.c b/upb/decode_fast.c index 5e34c8d931..1b9e6ead3f 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -3,30 +3,34 @@ #include "upb/port_def.inc" -#define UPB_PARSE_PARAMS \ - upb_decstate *d, const char *ptr, upb_msg *msg, upb_fasttable *table, \ +#define UPB_PARSE_PARAMS \ + upb_decstate *d, const char *ptr, upb_msg *msg, const upb_msglayout *table, \ uint64_t hasbits, uint64_t data #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data -const char *fastdecode_err(upb_decstate *d); - -const char *fastdecode_reallocarr(upb_decstate *d, const char *ptr, - upb_msg *msg, upb_fasttable *table, - size_t needbytes); - UPB_NOINLINE -static const char *fastdecode_dispatch(upb_decstate *d, const char *ptr, - upb_msg *msg, upb_fasttable *table, - uint64_t hasbits) { +const char *fastdecode_dispatch(upb_decstate *d, const char *ptr, upb_msg *msg, + const upb_msglayout *table, uint64_t hasbits) { uint16_t tag; uint64_t data; + size_t idx; if (UPB_UNLIKELY(ptr >= d->fastlimit)) return ptr; memcpy(&tag, ptr, 2); - data = table->field_data[(tag & 0xf7) >> 3] ^ tag; - return table->field_parser[(tag & 0xf7) >> 3](UPB_PARSE_ARGS); + idx = (tag & 0xf8) >> 3; + data = table->field_data[idx] ^ tag; + return table->field_parser[idx](UPB_PARSE_ARGS); } +#if 0 + +const char *fastdecode_err(upb_decstate *d); + +const char *fastdecode_reallocarr(upb_decstate *d, const char *ptr, + upb_msg *msg, upb_msglayout *table, + size_t needbytes); + + UPB_FORCEINLINE static bool fastdecode_checktag(uint64_t data, int tagbytes) { if (tagbytes == 1) { @@ -134,7 +138,7 @@ static void *fastdecode_getfield(upb_decstate *d, const char *ptr, upb_msg *msg, UPB_NOINLINE static const char *fastdecode_longstring(upb_decstate *d, const char *ptr, - upb_msg *msg, upb_fasttable *table, + upb_msg *msg, upb_msglayout *table, uint64_t hasbits, upb_strview *dst) { uint32_t len; if (!fastdecode_readlongsize(ptr, &len, &ptr)) { @@ -271,7 +275,7 @@ again: UPB_NOINLINE const char *fastdecode_longfixedpacked(upb_decstate *d, const char *ptr, - upb_msg *msg, upb_fasttable *table, + upb_msg *msg, upb_msglayout *table, uint64_t hasbits, void *dst); UPB_FORCEINLINE @@ -345,9 +349,6 @@ const char *upb_prf8_1bt(UPB_PARSE_PARAMS) { return fastdecode_fixed(UPB_PARSE_ARGS, 1, 8, CARD_r, &upb_ppf8_1bt); } -#if 0 - - // Generate all fixed functions. // {s,o,r,p} x {f4,f8} x {1bt,2bt} diff --git a/upb/def.c b/upb/def.c index 00a5877bde..c5225a1806 100644 --- a/upb/def.c +++ b/upb/def.c @@ -949,6 +949,7 @@ static bool make_layout(const upb_symtab *symtab, const upb_msgdef *m) { const upb_msglayout **submsgs; upb_msglayout_field *fields; upb_alloc *alloc = upb_arena_alloc(symtab->arena); + size_t i; memset(l, 0, sizeof(*l)); @@ -965,6 +966,10 @@ static bool make_layout(const upb_symtab *symtab, const upb_msgdef *m) { l->fields = fields; l->submsgs = submsgs; + for (i = 0; i < 32; i++) { + l->field_parser[i] = &fastdecode_generic; + } + if (upb_msgdef_mapentry(m)) { /* TODO(haberman): refactor this method so this special case is more * elegant. */ diff --git a/upb/msg.h b/upb/msg.h index 695c278b21..7a1ec760ed 100644 --- a/upb/msg.h +++ b/upb/msg.h @@ -46,7 +46,17 @@ typedef struct { uint8_t label; /* google.protobuf.Label or _UPB_LABEL_* above. */ } upb_msglayout_field; +struct upb_msglayout; +struct upb_decstate; + +typedef const char *_upb_field_parser(struct upb_decstate *d, const char *ptr, + upb_msg *msg, + const struct upb_msglayout *table, + uint64_t hasbits, uint64_t data); + typedef struct upb_msglayout { + _upb_field_parser *field_parser[32]; + uint64_t field_data[32]; const struct upb_msglayout *const* submsgs; const upb_msglayout_field *fields; /* Must be aligned to sizeof(void*). Doesn't include internal members like diff --git a/upbc/generator.cc b/upbc/generator.cc index 72df0244aa..cd247a2ad7 100644 --- a/upbc/generator.cc +++ b/upbc/generator.cc @@ -804,6 +804,14 @@ void WriteSource(const protobuf::FileDescriptor* file, Output& output) { } output("const upb_msglayout $0 = {\n", MessageInit(message)); + output(" {\n"); + for (int i = 0; i < 32; i++) { + output(" &fastdecode_generic,\n"); + } + output(" },\n"); + output(" {\n"); + output(" 0\n"); + output(" },\n"); output(" $0,\n", submsgs_array_ref); output(" $0,\n", fields_array_ref); output(" $0, $1, $2,\n", GetSizeInit(layout.message_size()),