From 021db6fcd57f2e27fe5801d55771d19464c31929 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sun, 25 Oct 2020 10:44:39 -0700 Subject: [PATCH] Allow larger tags into the table if they are unique mod 31. Also fixed a bug with fixed packed in decode_fast.c. --- cmake/google/protobuf/descriptor.upb.c | 30 ++++++++-------- upb/decode_fast.c | 1 + upbc/generator.cc | 48 ++++++++++++++++++-------- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/cmake/google/protobuf/descriptor.upb.c b/cmake/google/protobuf/descriptor.upb.c index 4129350f81..60e0c42f8c 100644 --- a/cmake/google/protobuf/descriptor.upb.c +++ b/cmake/google/protobuf/descriptor.upb.c @@ -299,7 +299,7 @@ const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit = { {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x000000003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, @@ -724,16 +724,16 @@ const upb_msglayout google_protobuf_FileOptions_msginit = { {0x001800000b00000a, &upb_pss_1bt}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, + {0x004800000e0002a2, &upb_pss_2bt}, + {0x005800000f0002aa, &upb_pss_2bt}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x00680000100002ba, &upb_pss_2bt}, {0x002800000c000042, &upb_pss_1bt}, {0x0004000001000048, &upb_psv4_1bt}, {0x0008000002000050, &upb_psb1_1bt}, {0x003800000d00005a, &upb_pss_1bt}, - {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x00980000130002e2, &upb_pss_2bt}, + {0x00a80000140002ea, &upb_pss_2bt}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0009000003000180, &upb_psb1_2bt}, @@ -833,7 +833,7 @@ const upb_msglayout google_protobuf_FieldOptions_msginit = { {0x0000000000000000, &fastdecode_generic}, {0x000e000005000028, &upb_psb1_1bt}, {0x0008000002000030, &upb_psv4_1bt}, - {0x0000000000000000, &fastdecode_generic}, + {0x001000003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x000f000006000050, &upb_psb1_1bt}, @@ -881,7 +881,7 @@ const upb_msglayout google_protobuf_OneofOptions_msginit = { {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x000000003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, @@ -931,7 +931,7 @@ const upb_msglayout google_protobuf_EnumOptions_msginit = { {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x000800003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, @@ -980,7 +980,7 @@ const upb_msglayout google_protobuf_EnumValueOptions_msginit = { {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x000800003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, @@ -1023,13 +1023,13 @@ static const upb_msglayout_field google_protobuf_ServiceOptions__fields[2] = { const upb_msglayout google_protobuf_ServiceOptions_msginit = { { {0x0000000000000000, &fastdecode_generic}, + {0x0001000001000288, &upb_psb1_2bt}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x000800003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, @@ -1073,13 +1073,13 @@ static const upb_msglayout_field google_protobuf_MethodOptions__fields[3] = { const upb_msglayout google_protobuf_MethodOptions_msginit = { { {0x0000000000000000, &fastdecode_generic}, + {0x0008000002000288, &upb_psb1_2bt}, + {0x0004000001000290, &upb_psv4_2bt}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, - {0x0000000000000000, &fastdecode_generic}, + {0x001000003f003eba, &upb_prm_2bt_max128b}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, {0x0000000000000000, &fastdecode_generic}, diff --git a/upb/decode_fast.c b/upb/decode_fast.c index e2252ed54f..7590b816eb 100644 --- a/upb/decode_fast.c +++ b/upb/decode_fast.c @@ -588,6 +588,7 @@ static const char *fastdecode_packedfixed(UPB_PARSE_PARAMS, int tagbytes, } } + ptr += tagbytes; int size = (uint8_t)ptr[0]; ptr++; if (size & 0x80) { diff --git a/upbc/generator.cc b/upbc/generator.cc index 6005e25146..4a6cce2a7e 100644 --- a/upbc/generator.cc +++ b/upbc/generator.cc @@ -739,13 +739,13 @@ struct SubmsgArray { typedef std::pair TableEntry; void TryFillTableEntry(const protobuf::Descriptor* message, - const MessageLayout& layout, int num, TableEntry& ent) { - const protobuf::FieldDescriptor* field = message->FindFieldByNumber(num); - if (!field) return; - + const MessageLayout& layout, + const protobuf::FieldDescriptor* field, + TableEntry& ent) { std::string type = ""; std::string cardinality = ""; - uint8_t wire_type = 0; + protobuf::internal::WireFormatLite::WireType wire_type = + protobuf::internal::WireFormatLite::WIRETYPE_VARINT; switch (field->type()) { case protobuf::FieldDescriptor::TYPE_BOOL: type = "b1"; @@ -763,13 +763,13 @@ void TryFillTableEntry(const protobuf::Descriptor* message, case protobuf::FieldDescriptor::TYPE_SFIXED32: case protobuf::FieldDescriptor::TYPE_FLOAT: type = "f4"; - wire_type = 5; + wire_type = protobuf::internal::WireFormatLite::WIRETYPE_FIXED32; break; case protobuf::FieldDescriptor::TYPE_FIXED64: case protobuf::FieldDescriptor::TYPE_SFIXED64: case protobuf::FieldDescriptor::TYPE_DOUBLE: type = "f8"; - wire_type = 1; + wire_type = protobuf::internal::WireFormatLite::WIRETYPE_FIXED64; break; case protobuf::FieldDescriptor::TYPE_SINT32: type = "z4"; @@ -780,14 +780,14 @@ void TryFillTableEntry(const protobuf::Descriptor* message, case protobuf::FieldDescriptor::TYPE_STRING: case protobuf::FieldDescriptor::TYPE_BYTES: type = "s"; - wire_type = 2; + wire_type = protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED; break; case protobuf::FieldDescriptor::TYPE_MESSAGE: if (field->is_map()) { return; // Not supported yet (ever?). } type = "m"; - wire_type = 2; + wire_type = protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED; break; default: return; // Not supported yet. @@ -797,7 +797,8 @@ void TryFillTableEntry(const protobuf::Descriptor* message, case protobuf::FieldDescriptor::LABEL_REPEATED: if (field->is_packed()) { cardinality = "p"; - wire_type = 2; + wire_type = + protobuf::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED; } else { cardinality = "r"; } @@ -812,8 +813,17 @@ void TryFillTableEntry(const protobuf::Descriptor* message, break; } - uint16_t expected_tag = (num << 3) | wire_type; - if (num > 15) expected_tag |= 0x100; + uint32_t unencoded_tag = + protobuf::internal::WireFormatLite::MakeTag(field->number(), wire_type); + uint8_t tag_bytes[10] = {0}; + protobuf::io::CodedOutputStream::WriteVarint32ToArray(unencoded_tag, + tag_bytes); + uint64_t expected_tag = 0; + memcpy(&expected_tag, tag_bytes, sizeof(expected_tag)); + if (expected_tag > 0x7fff) { + // Tag is >2 bytes. + return; + } MessageLayout::Size offset = layout.GetFieldOffset(field); // Data is: @@ -869,11 +879,11 @@ void TryFillTableEntry(const protobuf::Descriptor* message, } } ent.first = absl::Substitute("upb_p$0$1_$2bt_max$3b", cardinality, type, - (num > 15) ? "2" : "1", size_ceil); + expected_tag > 0xff ? "2" : "1", size_ceil); } else { ent.first = absl::Substitute("upb_p$0$1_$2bt", cardinality, type, - (num > 15) ? "2" : "1"); + expected_tag > 0xff ? "2" : "1"); } ent.second = data; } @@ -883,7 +893,15 @@ std::vector FastDecodeTable(const protobuf::Descriptor* message, std::vector table; for (int i = 0; i < 32; i++) { table.emplace_back(TableEntry{"fastdecode_generic", 0}); - TryFillTableEntry(message, layout, i, table.back()); + } + for (int i = 0; i < message->field_count(); i++) { + const protobuf::FieldDescriptor* field = message->field(i); + int slot = field->number() & 31; + if (table[slot].first != "fastdecode_generic") { + // This slot is already populated by another field. + continue; + } + TryFillTableEntry(message, layout, field, table[slot]); } return table; }