From e3b0943412adf3b6a3ad4bb52d5e17ef7966b361 Mon Sep 17 00:00:00 2001 From: Sandy Zhang Date: Mon, 7 Aug 2023 09:30:02 -0700 Subject: [PATCH] Update protobuf's upb dependency to current main HEAD. PiperOrigin-RevId: 554500222 --- php/ext/google/protobuf/php-upb.c | 53 ++++++-- php/ext/google/protobuf/php-upb.h | 170 +++++++++++++------------- protobuf_deps.bzl | 4 +- ruby/ext/google/protobuf_c/ruby-upb.c | 53 ++++++-- ruby/ext/google/protobuf_c/ruby-upb.h | 170 +++++++++++++------------- 5 files changed, 260 insertions(+), 190 deletions(-) diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 8300206a86..9e5a82e1e6 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -249,8 +249,23 @@ * expected behavior. */ -#if defined(__SANITIZE_ADDRESS__) +/* Due to preprocessor limitations, the conditional logic for setting + * UPN_CLANG_ASAN below cannot be consolidated into a portable one-liner. + * See https://gcc.gnu.org/onlinedocs/cpp/_005f_005fhas_005fattribute.html. + */ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define UPB_CLANG_ASAN 1 +#else +#define UPB_CLANG_ASAN 0 +#endif +#else +#define UPB_CLANG_ASAN 0 +#endif + +#if defined(__SANITIZE_ADDRESS__) || UPB_CLANG_ASAN #define UPB_ASAN 1 +#define UPB_ASAN_GUARD_SIZE 32 #ifdef __cplusplus extern "C" { #endif @@ -265,6 +280,7 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); __asan_unpoison_memory_region((addr), (size)) #else #define UPB_ASAN 0 +#define UPB_ASAN_GUARD_SIZE 0 #define UPB_POISON_MEMORY_REGION(addr, size) \ ((void)(addr), (void)(size)) #define UPB_UNPOISON_MEMORY_REGION(addr, size) \ @@ -5780,6 +5796,7 @@ static void* upb_global_allocfunc(upb_alloc* alloc, void* ptr, size_t oldsize, upb_alloc upb_alloc_global = {&upb_global_allocfunc}; + // Must be last. struct _upb_MemBlock { @@ -5818,7 +5835,7 @@ static _upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) { // // This is true because: // - If no fuses occur, this will eventually become the root. - // - If fuses are actively occuring, the root may change, but the + // - If fuses are actively occurring, the root may change, but the // invariant is that `parent_or_count` merely points to *a* parent. // // In other words, it is moving towards "the" root, and that root may move @@ -8164,14 +8181,16 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } - } else if (has_type_name) { - f->type_ = - UPB_FIELD_TYPE_UNSPECIFIED; // We'll assign this in resolve_fielddef() } - if (f->type_ < kUpb_FieldType_Double || f->type_ > kUpb_FieldType_SInt64) { - _upb_DefBuilder_Errf(ctx, "invalid type for field %s (%d)", f->full_name, - f->type_); + if (!has_type && has_type_name) { + f->type_ = + UPB_FIELD_TYPE_UNSPECIFIED; // We'll assign this in resolve_subdef() + } else { + if (f->type_ < kUpb_FieldType_Double || f->type_ > kUpb_FieldType_SInt64) { + _upb_DefBuilder_Errf(ctx, "invalid type for field %s (%d)", f->full_name, + f->type_); + } } if (f->label_ < kUpb_Label_Optional || f->label_ > kUpb_Label_Repeated) { @@ -8220,14 +8239,15 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->is_packed = UPB_DESC(FieldOptions_packed)(f->opts); } else { // Repeated fields default to packed for proto3 only. - f->is_packed = upb_FieldDef_IsPrimitive(f) && + f->is_packed = has_type && upb_FieldDef_IsPrimitive(f) && f->label_ == kUpb_Label_Repeated && upb_FileDef_Syntax(f->file) == kUpb_Syntax_Proto3; } f->has_presence = (!upb_FieldDef_IsRepeated(f)) && - (upb_FieldDef_IsSubMessage(f) || upb_FieldDef_ContainingOneof(f) || + (f->type_ == kUpb_FieldType_Message || f->type_ == kUpb_FieldType_Group || + upb_FieldDef_ContainingOneof(f) || (upb_FileDef_Syntax(f->file) == kUpb_Syntax_Proto2)); } @@ -8335,16 +8355,22 @@ static void resolve_subdef(upb_DefBuilder* ctx, const char* prefix, case UPB_DEFTYPE_ENUM: f->sub.enumdef = def; f->type_ = kUpb_FieldType_Enum; + if (!UPB_DESC(FieldOptions_has_packed)(f->opts)) { + f->is_packed = f->label_ == kUpb_Label_Repeated && + upb_FileDef_Syntax(f->file) == kUpb_Syntax_Proto3; + } break; case UPB_DEFTYPE_MSG: f->sub.msgdef = def; f->type_ = kUpb_FieldType_Message; // It appears there is no way of // this being a group. + f->has_presence = !upb_FieldDef_IsRepeated(f); break; default: _upb_DefBuilder_Errf(ctx, "Couldn't resolve type name for field %s", f->full_name); } + break; } case kUpb_FieldType_Message: case kUpb_FieldType_Group: @@ -9955,7 +9981,8 @@ void _upb_OneofDef_Insert(upb_DefBuilder* ctx, upb_OneofDef* o, // TODO(salo): More redundant work happening here. const bool name_exists = upb_strtable_lookup2(&o->ntof, name, size, NULL); if (UPB_UNLIKELY(name_exists)) { - _upb_DefBuilder_Errf(ctx, "oneof fields have the same name (%s)", name); + _upb_DefBuilder_Errf(ctx, "oneof fields have the same name (%.*s)", + (int)size, name); } const bool ok = upb_inttable_insert(&o->itof, number, v, ctx->arena) && @@ -14479,7 +14506,7 @@ bool upb_MiniTable_Link(upb_MiniTable* mt, const upb_MiniTable** sub_tables, for (int i = 0; i < mt->field_count; i++) { upb_MiniTableField* f = (upb_MiniTableField*)&mt->fields[i]; - if (upb_MiniTableField_CType(f) == kUpb_CType_Enum) { + if (upb_MiniTableField_IsClosedEnum(f)) { const upb_MiniTableEnum* sub = sub_enums[enum_count++]; if (enum_count > sub_enum_count) return false; if (sub != NULL) { @@ -14673,6 +14700,8 @@ bool upb_MiniTable_NextOneofField(const upb_MiniTable* m, #undef UPB_POISON_MEMORY_REGION #undef UPB_UNPOISON_MEMORY_REGION #undef UPB_ASAN +#undef UPB_ASAN_GUARD_SIZE +#undef UPB_CLANG_ASAN #undef UPB_TREAT_PROTO2_ENUMS_LIKE_PROTO3 #undef UPB_DEPRECATED #undef UPB_GNUC_MIN diff --git a/php/ext/google/protobuf/php-upb.h b/php/ext/google/protobuf/php-upb.h index eabbd8b3ff..d56d7f714f 100644 --- a/php/ext/google/protobuf/php-upb.h +++ b/php/ext/google/protobuf/php-upb.h @@ -248,8 +248,23 @@ * expected behavior. */ -#if defined(__SANITIZE_ADDRESS__) +/* Due to preprocessor limitations, the conditional logic for setting + * UPN_CLANG_ASAN below cannot be consolidated into a portable one-liner. + * See https://gcc.gnu.org/onlinedocs/cpp/_005f_005fhas_005fattribute.html. + */ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define UPB_CLANG_ASAN 1 +#else +#define UPB_CLANG_ASAN 0 +#endif +#else +#define UPB_CLANG_ASAN 0 +#endif + +#if defined(__SANITIZE_ADDRESS__) || UPB_CLANG_ASAN #define UPB_ASAN 1 +#define UPB_ASAN_GUARD_SIZE 32 #ifdef __cplusplus extern "C" { #endif @@ -264,6 +279,7 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); __asan_unpoison_memory_region((addr), (size)) #else #define UPB_ASAN 0 +#define UPB_ASAN_GUARD_SIZE 0 #define UPB_POISON_MEMORY_REGION(addr, size) \ ((void)(addr), (void)(size)) #define UPB_UNPOISON_MEMORY_REGION(addr, size) \ @@ -486,14 +502,22 @@ UPB_INLINE bool upb_StringView_IsEqual(upb_StringView a, upb_StringView b) { #include + +#ifndef UPB_MESSAGE_TYPEDEF_H_ +#define UPB_MESSAGE_TYPEDEF_H_ + +// This typedef needs its own header to resolve a circular dependency between +// messages and mini tables. +typedef void upb_Message; + +#endif /* UPB_MESSAGE_TYPEDEF_H_ */ + // Must be last. #ifdef __cplusplus extern "C" { #endif -typedef void upb_Message; - // When a upb_Message* is stored in a message, array, or map, it is stored in a // tagged form. If the tag bit is set, the referenced upb_Message is of type // _kUpb_MiniTable_Empty (a sentinel message type with no fields) instead of @@ -625,8 +649,6 @@ UPB_INLINE bool upb_MiniTableEnum_CheckValue(const struct upb_MiniTableEnum* e, // Must be last. -// LINT.IfChange(mini_table_field_layout) - struct upb_MiniTableField { uint32_t number; uint16_t offset; @@ -679,8 +701,6 @@ typedef enum { #define kUpb_FieldRep_Shift 6 -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/mini_table_field.ts:mini_table_field_layout) - UPB_INLINE upb_FieldRep _upb_MiniTableField_GetRep(const struct upb_MiniTableField* field) { return (upb_FieldRep)(field->mode >> kUpb_FieldRep_Shift); @@ -732,8 +752,6 @@ UPB_INLINE bool upb_IsSubMessage(const struct upb_MiniTableField* field) { // Must be last. -typedef void upb_Message; - struct upb_Decoder; typedef const char* _upb_FieldParser(struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, @@ -755,8 +773,6 @@ typedef enum { kUpb_ExtMode_IsMapEntry = 4, } upb_ExtMode; -// LINT.IfChange(mini_table_layout) - union upb_MiniTableSub; // upb_MiniTable represents the memory layout of a given upb_MessageDef. @@ -782,8 +798,6 @@ struct upb_MiniTable { _upb_FastTable_Entry fasttable[]; }; -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/mini_table.ts:presence_logic) - #ifdef __cplusplus extern "C" { #endif @@ -1100,14 +1114,10 @@ UPB_INLINE void upb_gfree(void* ptr) { upb_free(&upb_alloc_global, ptr); } typedef struct upb_Arena upb_Arena; -// LINT.IfChange(arena_head) - typedef struct { char *ptr, *end; } _upb_ArenaHead; -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/arena.ts:arena_head) - #ifdef __cplusplus extern "C" { #endif @@ -1131,7 +1141,8 @@ UPB_INLINE size_t _upb_ArenaHas(upb_Arena* a) { UPB_API_INLINE void* upb_Arena_Malloc(upb_Arena* a, size_t size) { size = UPB_ALIGN_MALLOC(size); - if (UPB_UNLIKELY(_upb_ArenaHas(a) < size)) { + size_t span = size + UPB_ASAN_GUARD_SIZE; + if (UPB_UNLIKELY(_upb_ArenaHas(a) < span)) { return _upb_Arena_SlowMalloc(a, size); } @@ -1142,18 +1153,7 @@ UPB_API_INLINE void* upb_Arena_Malloc(upb_Arena* a, size_t size) { UPB_ASSERT(UPB_ALIGN_MALLOC(size) == size); UPB_UNPOISON_MEMORY_REGION(ret, size); - h->ptr += size; - -#if UPB_ASAN - { - size_t guard_size = 32; - if (_upb_ArenaHas(a) >= guard_size) { - h->ptr += guard_size; - } else { - h->ptr = h->end; - } - } -#endif + h->ptr += span; return ret; } @@ -1167,7 +1167,8 @@ UPB_API_INLINE void upb_Arena_ShrinkLast(upb_Arena* a, void* ptr, _upb_ArenaHead* h = (_upb_ArenaHead*)a; oldsize = UPB_ALIGN_MALLOC(oldsize); size = UPB_ALIGN_MALLOC(size); - UPB_ASSERT((char*)ptr + oldsize == h->ptr); // Must be the last alloc. + // Must be the last alloc. + UPB_ASSERT((char*)ptr + oldsize == h->ptr - UPB_ASAN_GUARD_SIZE); UPB_ASSERT(size <= oldsize); h->ptr = (char*)ptr + size; } @@ -1930,7 +1931,7 @@ UPB_INLINE int upb_Log2Ceiling(int x) { return 32 - __builtin_clz(x - 1); #else int lg2 = 0; - while (1 << lg2 < x) lg2++; + while ((1 << lg2) < x) lg2++; return lg2; #endif } @@ -2099,7 +2100,15 @@ typedef struct { } upb_MapEntryData; typedef struct { - void* internal_data; + // LINT.IfChange(internal_layout) + union { + void* internal_data; + + // Force 8-byte alignment, since the data members may contain members that + // require 8-byte alignment. + double d; + }; + // LINT.ThenChange(//depot/google3/third_party/upb/upb/message/internal.h:internal_layout) upb_MapEntryData data; } upb_MapEntry; @@ -2267,10 +2276,22 @@ UPB_INLINE void _upb_msg_map_set_value(void* msg, const void* val, #endif /* UPB_COLLECTIONS_MAP_GENCODE_UTIL_H_ */ -// This header is deprecated, use upb/mini_table/extension_registry.h instead +#ifndef UPB_MESSAGE_ACCESSORS_H_ +#define UPB_MESSAGE_ACCESSORS_H_ + + +/* +** Our memory representation for parsing tables and messages themselves. +** Functions in this file are used by generated code and possibly reflection. +** +** The definitions in this file are internal to upb. +**/ + +#ifndef UPB_MESSAGE_INTERNAL_H_ +#define UPB_MESSAGE_INTERNAL_H_ -#ifndef UPB_EXTENSION_REGISTRY_H_ -#define UPB_EXTENSION_REGISTRY_H_ +#include +#include #ifndef UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ @@ -2347,30 +2368,6 @@ UPB_API const upb_MiniTableExtension* upb_ExtensionRegistry_Lookup( #endif /* UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ */ -#endif /* UPB_EXTENSION_REGISTRY_H_ */ - -#ifndef UPB_MESSAGE_ACCESSORS_H_ -#define UPB_MESSAGE_ACCESSORS_H_ - - -#ifndef UPB_MESSAGE_ACCESSORS_INTERNAL_H_ -#define UPB_MESSAGE_ACCESSORS_INTERNAL_H_ - - -/* -** Our memory representation for parsing tables and messages themselves. -** Functions in this file are used by generated code and possibly reflection. -** -** The definitions in this file are internal to upb. -**/ - -#ifndef UPB_MESSAGE_INTERNAL_H_ -#define UPB_MESSAGE_INTERNAL_H_ - -#include -#include - - // Must be last. #ifdef __cplusplus @@ -2411,7 +2408,15 @@ typedef struct { } upb_Message_InternalData; typedef struct { - upb_Message_InternalData* internal; + // LINT.IfChange(internal_layout) + union { + upb_Message_InternalData* internal; + + // Force 8-byte alignment, since the data members may contain members that + // require 8-byte alignment. + double d; + }; + // LINT.ThenChange(//depot/google3/third_party/upb/upb/message/internal/map_entry.h:internal_layout) /* Message data follows. */ } upb_Message_Internal; @@ -2454,6 +2459,10 @@ bool _upb_Message_AddUnknown(upb_Message* msg, const char* data, size_t len, #endif /* UPB_MESSAGE_INTERNAL_H_ */ +#ifndef UPB_MESSAGE_INTERNAL_ACCESSORS_H_ +#define UPB_MESSAGE_INTERNAL_ACCESSORS_H_ + + // Must be last. #if defined(__GNUC__) && !defined(__clang__) @@ -2532,7 +2541,6 @@ UPB_INLINE uint32_t _upb_getoneofcase_field(const upb_Message* msg, } // LINT.ThenChange(GoogleInternalName2) -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/presence.ts:presence_logic) UPB_INLINE bool _upb_MiniTableField_InOneOf(const upb_MiniTableField* field) { return field->presence < 0; @@ -2557,8 +2565,6 @@ UPB_INLINE void _upb_Message_SetPresence(upb_Message* msg, } } -// LINT.IfChange(message_raw_fields) - UPB_INLINE bool _upb_MiniTable_ValueIsNonZero(const void* default_val, const upb_MiniTableField* field) { char zero[16] = {0}; @@ -2597,8 +2603,6 @@ UPB_INLINE void _upb_MiniTable_CopyFieldData(void* to, const void* from, UPB_UNREACHABLE(); } -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/message.ts:message_raw_fields) - UPB_INLINE size_t _upb_MiniTable_ElementSizeLg2(const upb_MiniTableField* field) { const unsigned char table[] = { @@ -2807,7 +2811,7 @@ UPB_INLINE upb_Map* _upb_Message_GetOrCreateMutableMap( #endif -#endif // UPB_MESSAGE_ACCESSORS_INTERNAL_H_ +#endif // UPB_MESSAGE_INTERNAL_ACCESSORS_H_ // Must be last. @@ -3393,9 +3397,9 @@ upb_MiniTable* upb_MiniTable_BuildWithBuf(const char* data, size_t len, // Must be last. struct upb_MiniTableFile { - const upb_MiniTable** msgs; - const upb_MiniTableEnum** enums; - const upb_MiniTableExtension** exts; + const struct upb_MiniTable** msgs; + const struct upb_MiniTableEnum** enums; + const struct upb_MiniTableExtension** exts; int msg_count; int enum_count; int ext_count; @@ -11527,8 +11531,8 @@ double _upb_NoLocaleStrtod(const char *str, char **endptr); #endif /* UPB_LEX_STRTOD_H_ */ -#ifndef UPB_MEM_ARENA_INTERNAL_H_ -#define UPB_MEM_ARENA_INTERNAL_H_ +#ifndef UPB_MEM_INTERNAL_ARENA_H_ +#define UPB_MEM_INTERNAL_ARENA_H_ // Must be last. @@ -11610,7 +11614,7 @@ UPB_INLINE bool upb_Arena_HasInitialBlock(upb_Arena* arena) { } -#endif /* UPB_MEM_ARENA_INTERNAL_H_ */ +#endif /* UPB_MEM_INTERNAL_ARENA_H_ */ #ifndef UPB_PORT_ATOMIC_H_ #define UPB_PORT_ATOMIC_H_ @@ -11706,8 +11710,8 @@ extern "C" { #define UPB_WIRE_READER_H_ -#ifndef UPB_WIRE_SWAP_INTERNAL_H_ -#define UPB_WIRE_SWAP_INTERNAL_H_ +#ifndef UPB_WIRE_INTERNAL_SWAP_H_ +#define UPB_WIRE_INTERNAL_SWAP_H_ // Must be last. @@ -11739,7 +11743,7 @@ UPB_INLINE uint64_t _upb_BigEndian_Swap64(uint64_t val) { #endif -#endif /* UPB_WIRE_SWAP_INTERNAL_H_ */ +#endif /* UPB_WIRE_INTERNAL_SWAP_H_ */ #ifndef UPB_WIRE_TYPES_H_ #define UPB_WIRE_TYPES_H_ @@ -12572,8 +12576,8 @@ upb_MethodDef* _upb_MethodDefs_New( #endif /* UPB_REFLECTION_METHOD_DEF_INTERNAL_H_ */ -#ifndef UPB_WIRE_COMMON_INTERNAL_H_ -#define UPB_WIRE_COMMON_INTERNAL_H_ +#ifndef UPB_WIRE_INTERNAL_COMMON_H_ +#define UPB_WIRE_INTERNAL_COMMON_H_ // Must be last. @@ -12592,15 +12596,15 @@ enum { }; -#endif /* UPB_WIRE_COMMON_INTERNAL_H_ */ +#endif /* UPB_WIRE_INTERNAL_COMMON_H_ */ /* * Internal implementation details of the decoder that are shared between * decode.c and decode_fast.c. */ -#ifndef UPB_WIRE_DECODE_INTERNAL_H_ -#define UPB_WIRE_DECODE_INTERNAL_H_ +#ifndef UPB_WIRE_INTERNAL_DECODE_H_ +#define UPB_WIRE_INTERNAL_DECODE_H_ #include "utf8_range.h" @@ -12723,7 +12727,7 @@ UPB_INLINE uint32_t _upb_FastDecoder_LoadTag(const char* ptr) { } -#endif /* UPB_WIRE_DECODE_INTERNAL_H_ */ +#endif /* UPB_WIRE_INTERNAL_DECODE_H_ */ #ifndef UPB_MINI_DESCRIPTOR_INTERNAL_BASE92_H_ #define UPB_MINI_DESCRIPTOR_INTERNAL_BASE92_H_ @@ -12919,6 +12923,8 @@ UPB_INLINE const char* upb_MdDecoder_DecodeBase92Varint( #undef UPB_POISON_MEMORY_REGION #undef UPB_UNPOISON_MEMORY_REGION #undef UPB_ASAN +#undef UPB_ASAN_GUARD_SIZE +#undef UPB_CLANG_ASAN #undef UPB_TREAT_PROTO2_ENUMS_LIKE_PROTO3 #undef UPB_DEPRECATED #undef UPB_GNUC_MIN diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index e3cd4e1195..1c6e8f652b 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -150,7 +150,7 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "01fed1cc1ba255bf22b49393ba054b8d270e6ba3", - sha256 = "387bef0d61094773a4ce7dd1c3d92bb99444155531e00945161ec9cd36e5bfce", + commit = "58877b55e0796bcca743e9bd4d2be42092562f30", + sha256 = "b3a3279cffb91e22fd38fb316a39ded87211bf89ff39337c7f151d88ed3b71fd", patches = ["@com_google_protobuf//build_defs:upb.patch"], ) diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 5ed3c3d3fe..a2dd07e35e 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -249,8 +249,23 @@ * expected behavior. */ -#if defined(__SANITIZE_ADDRESS__) +/* Due to preprocessor limitations, the conditional logic for setting + * UPN_CLANG_ASAN below cannot be consolidated into a portable one-liner. + * See https://gcc.gnu.org/onlinedocs/cpp/_005f_005fhas_005fattribute.html. + */ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define UPB_CLANG_ASAN 1 +#else +#define UPB_CLANG_ASAN 0 +#endif +#else +#define UPB_CLANG_ASAN 0 +#endif + +#if defined(__SANITIZE_ADDRESS__) || UPB_CLANG_ASAN #define UPB_ASAN 1 +#define UPB_ASAN_GUARD_SIZE 32 #ifdef __cplusplus extern "C" { #endif @@ -265,6 +280,7 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); __asan_unpoison_memory_region((addr), (size)) #else #define UPB_ASAN 0 +#define UPB_ASAN_GUARD_SIZE 0 #define UPB_POISON_MEMORY_REGION(addr, size) \ ((void)(addr), (void)(size)) #define UPB_UNPOISON_MEMORY_REGION(addr, size) \ @@ -5319,6 +5335,7 @@ static void* upb_global_allocfunc(upb_alloc* alloc, void* ptr, size_t oldsize, upb_alloc upb_alloc_global = {&upb_global_allocfunc}; + // Must be last. struct _upb_MemBlock { @@ -5357,7 +5374,7 @@ static _upb_ArenaRoot _upb_Arena_FindRoot(upb_Arena* a) { // // This is true because: // - If no fuses occur, this will eventually become the root. - // - If fuses are actively occuring, the root may change, but the + // - If fuses are actively occurring, the root may change, but the // invariant is that `parent_or_count` merely points to *a* parent. // // In other words, it is moving towards "the" root, and that root may move @@ -7703,14 +7720,16 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->full_name, (int)f->type_); } } - } else if (has_type_name) { - f->type_ = - UPB_FIELD_TYPE_UNSPECIFIED; // We'll assign this in resolve_fielddef() } - if (f->type_ < kUpb_FieldType_Double || f->type_ > kUpb_FieldType_SInt64) { - _upb_DefBuilder_Errf(ctx, "invalid type for field %s (%d)", f->full_name, - f->type_); + if (!has_type && has_type_name) { + f->type_ = + UPB_FIELD_TYPE_UNSPECIFIED; // We'll assign this in resolve_subdef() + } else { + if (f->type_ < kUpb_FieldType_Double || f->type_ > kUpb_FieldType_SInt64) { + _upb_DefBuilder_Errf(ctx, "invalid type for field %s (%d)", f->full_name, + f->type_); + } } if (f->label_ < kUpb_Label_Optional || f->label_ > kUpb_Label_Repeated) { @@ -7759,14 +7778,15 @@ static void _upb_FieldDef_Create(upb_DefBuilder* ctx, const char* prefix, f->is_packed = UPB_DESC(FieldOptions_packed)(f->opts); } else { // Repeated fields default to packed for proto3 only. - f->is_packed = upb_FieldDef_IsPrimitive(f) && + f->is_packed = has_type && upb_FieldDef_IsPrimitive(f) && f->label_ == kUpb_Label_Repeated && upb_FileDef_Syntax(f->file) == kUpb_Syntax_Proto3; } f->has_presence = (!upb_FieldDef_IsRepeated(f)) && - (upb_FieldDef_IsSubMessage(f) || upb_FieldDef_ContainingOneof(f) || + (f->type_ == kUpb_FieldType_Message || f->type_ == kUpb_FieldType_Group || + upb_FieldDef_ContainingOneof(f) || (upb_FileDef_Syntax(f->file) == kUpb_Syntax_Proto2)); } @@ -7874,16 +7894,22 @@ static void resolve_subdef(upb_DefBuilder* ctx, const char* prefix, case UPB_DEFTYPE_ENUM: f->sub.enumdef = def; f->type_ = kUpb_FieldType_Enum; + if (!UPB_DESC(FieldOptions_has_packed)(f->opts)) { + f->is_packed = f->label_ == kUpb_Label_Repeated && + upb_FileDef_Syntax(f->file) == kUpb_Syntax_Proto3; + } break; case UPB_DEFTYPE_MSG: f->sub.msgdef = def; f->type_ = kUpb_FieldType_Message; // It appears there is no way of // this being a group. + f->has_presence = !upb_FieldDef_IsRepeated(f); break; default: _upb_DefBuilder_Errf(ctx, "Couldn't resolve type name for field %s", f->full_name); } + break; } case kUpb_FieldType_Message: case kUpb_FieldType_Group: @@ -9494,7 +9520,8 @@ void _upb_OneofDef_Insert(upb_DefBuilder* ctx, upb_OneofDef* o, // TODO(salo): More redundant work happening here. const bool name_exists = upb_strtable_lookup2(&o->ntof, name, size, NULL); if (UPB_UNLIKELY(name_exists)) { - _upb_DefBuilder_Errf(ctx, "oneof fields have the same name (%s)", name); + _upb_DefBuilder_Errf(ctx, "oneof fields have the same name (%.*s)", + (int)size, name); } const bool ok = upb_inttable_insert(&o->itof, number, v, ctx->arena) && @@ -14018,7 +14045,7 @@ bool upb_MiniTable_Link(upb_MiniTable* mt, const upb_MiniTable** sub_tables, for (int i = 0; i < mt->field_count; i++) { upb_MiniTableField* f = (upb_MiniTableField*)&mt->fields[i]; - if (upb_MiniTableField_CType(f) == kUpb_CType_Enum) { + if (upb_MiniTableField_IsClosedEnum(f)) { const upb_MiniTableEnum* sub = sub_enums[enum_count++]; if (enum_count > sub_enum_count) return false; if (sub != NULL) { @@ -14212,6 +14239,8 @@ bool upb_MiniTable_NextOneofField(const upb_MiniTable* m, #undef UPB_POISON_MEMORY_REGION #undef UPB_UNPOISON_MEMORY_REGION #undef UPB_ASAN +#undef UPB_ASAN_GUARD_SIZE +#undef UPB_CLANG_ASAN #undef UPB_TREAT_PROTO2_ENUMS_LIKE_PROTO3 #undef UPB_DEPRECATED #undef UPB_GNUC_MIN diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 2e48b2023d..41be731db1 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -250,8 +250,23 @@ * expected behavior. */ -#if defined(__SANITIZE_ADDRESS__) +/* Due to preprocessor limitations, the conditional logic for setting + * UPN_CLANG_ASAN below cannot be consolidated into a portable one-liner. + * See https://gcc.gnu.org/onlinedocs/cpp/_005f_005fhas_005fattribute.html. + */ +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define UPB_CLANG_ASAN 1 +#else +#define UPB_CLANG_ASAN 0 +#endif +#else +#define UPB_CLANG_ASAN 0 +#endif + +#if defined(__SANITIZE_ADDRESS__) || UPB_CLANG_ASAN #define UPB_ASAN 1 +#define UPB_ASAN_GUARD_SIZE 32 #ifdef __cplusplus extern "C" { #endif @@ -266,6 +281,7 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); __asan_unpoison_memory_region((addr), (size)) #else #define UPB_ASAN 0 +#define UPB_ASAN_GUARD_SIZE 0 #define UPB_POISON_MEMORY_REGION(addr, size) \ ((void)(addr), (void)(size)) #define UPB_UNPOISON_MEMORY_REGION(addr, size) \ @@ -488,14 +504,22 @@ UPB_INLINE bool upb_StringView_IsEqual(upb_StringView a, upb_StringView b) { #include + +#ifndef UPB_MESSAGE_TYPEDEF_H_ +#define UPB_MESSAGE_TYPEDEF_H_ + +// This typedef needs its own header to resolve a circular dependency between +// messages and mini tables. +typedef void upb_Message; + +#endif /* UPB_MESSAGE_TYPEDEF_H_ */ + // Must be last. #ifdef __cplusplus extern "C" { #endif -typedef void upb_Message; - // When a upb_Message* is stored in a message, array, or map, it is stored in a // tagged form. If the tag bit is set, the referenced upb_Message is of type // _kUpb_MiniTable_Empty (a sentinel message type with no fields) instead of @@ -627,8 +651,6 @@ UPB_INLINE bool upb_MiniTableEnum_CheckValue(const struct upb_MiniTableEnum* e, // Must be last. -// LINT.IfChange(mini_table_field_layout) - struct upb_MiniTableField { uint32_t number; uint16_t offset; @@ -681,8 +703,6 @@ typedef enum { #define kUpb_FieldRep_Shift 6 -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/mini_table_field.ts:mini_table_field_layout) - UPB_INLINE upb_FieldRep _upb_MiniTableField_GetRep(const struct upb_MiniTableField* field) { return (upb_FieldRep)(field->mode >> kUpb_FieldRep_Shift); @@ -734,8 +754,6 @@ UPB_INLINE bool upb_IsSubMessage(const struct upb_MiniTableField* field) { // Must be last. -typedef void upb_Message; - struct upb_Decoder; typedef const char* _upb_FieldParser(struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table, @@ -757,8 +775,6 @@ typedef enum { kUpb_ExtMode_IsMapEntry = 4, } upb_ExtMode; -// LINT.IfChange(mini_table_layout) - union upb_MiniTableSub; // upb_MiniTable represents the memory layout of a given upb_MessageDef. @@ -784,8 +800,6 @@ struct upb_MiniTable { _upb_FastTable_Entry fasttable[]; }; -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/mini_table.ts:presence_logic) - #ifdef __cplusplus extern "C" { #endif @@ -1102,14 +1116,10 @@ UPB_INLINE void upb_gfree(void* ptr) { upb_free(&upb_alloc_global, ptr); } typedef struct upb_Arena upb_Arena; -// LINT.IfChange(arena_head) - typedef struct { char *ptr, *end; } _upb_ArenaHead; -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/arena.ts:arena_head) - #ifdef __cplusplus extern "C" { #endif @@ -1133,7 +1143,8 @@ UPB_INLINE size_t _upb_ArenaHas(upb_Arena* a) { UPB_API_INLINE void* upb_Arena_Malloc(upb_Arena* a, size_t size) { size = UPB_ALIGN_MALLOC(size); - if (UPB_UNLIKELY(_upb_ArenaHas(a) < size)) { + size_t span = size + UPB_ASAN_GUARD_SIZE; + if (UPB_UNLIKELY(_upb_ArenaHas(a) < span)) { return _upb_Arena_SlowMalloc(a, size); } @@ -1144,18 +1155,7 @@ UPB_API_INLINE void* upb_Arena_Malloc(upb_Arena* a, size_t size) { UPB_ASSERT(UPB_ALIGN_MALLOC(size) == size); UPB_UNPOISON_MEMORY_REGION(ret, size); - h->ptr += size; - -#if UPB_ASAN - { - size_t guard_size = 32; - if (_upb_ArenaHas(a) >= guard_size) { - h->ptr += guard_size; - } else { - h->ptr = h->end; - } - } -#endif + h->ptr += span; return ret; } @@ -1169,7 +1169,8 @@ UPB_API_INLINE void upb_Arena_ShrinkLast(upb_Arena* a, void* ptr, _upb_ArenaHead* h = (_upb_ArenaHead*)a; oldsize = UPB_ALIGN_MALLOC(oldsize); size = UPB_ALIGN_MALLOC(size); - UPB_ASSERT((char*)ptr + oldsize == h->ptr); // Must be the last alloc. + // Must be the last alloc. + UPB_ASSERT((char*)ptr + oldsize == h->ptr - UPB_ASAN_GUARD_SIZE); UPB_ASSERT(size <= oldsize); h->ptr = (char*)ptr + size; } @@ -1932,7 +1933,7 @@ UPB_INLINE int upb_Log2Ceiling(int x) { return 32 - __builtin_clz(x - 1); #else int lg2 = 0; - while (1 << lg2 < x) lg2++; + while ((1 << lg2) < x) lg2++; return lg2; #endif } @@ -2101,7 +2102,15 @@ typedef struct { } upb_MapEntryData; typedef struct { - void* internal_data; + // LINT.IfChange(internal_layout) + union { + void* internal_data; + + // Force 8-byte alignment, since the data members may contain members that + // require 8-byte alignment. + double d; + }; + // LINT.ThenChange(//depot/google3/third_party/upb/upb/message/internal.h:internal_layout) upb_MapEntryData data; } upb_MapEntry; @@ -2231,10 +2240,22 @@ UPB_INLINE void _upb_msg_map_set_value(void* msg, const void* val, #endif /* UPB_COLLECTIONS_MAP_GENCODE_UTIL_H_ */ -// This header is deprecated, use upb/mini_table/extension_registry.h instead +#ifndef UPB_MESSAGE_ACCESSORS_H_ +#define UPB_MESSAGE_ACCESSORS_H_ + + +/* +** Our memory representation for parsing tables and messages themselves. +** Functions in this file are used by generated code and possibly reflection. +** +** The definitions in this file are internal to upb. +**/ + +#ifndef UPB_MESSAGE_INTERNAL_H_ +#define UPB_MESSAGE_INTERNAL_H_ -#ifndef UPB_EXTENSION_REGISTRY_H_ -#define UPB_EXTENSION_REGISTRY_H_ +#include +#include #ifndef UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ @@ -2311,30 +2332,6 @@ UPB_API const upb_MiniTableExtension* upb_ExtensionRegistry_Lookup( #endif /* UPB_MINI_TABLE_EXTENSION_REGISTRY_H_ */ -#endif /* UPB_EXTENSION_REGISTRY_H_ */ - -#ifndef UPB_MESSAGE_ACCESSORS_H_ -#define UPB_MESSAGE_ACCESSORS_H_ - - -#ifndef UPB_MESSAGE_ACCESSORS_INTERNAL_H_ -#define UPB_MESSAGE_ACCESSORS_INTERNAL_H_ - - -/* -** Our memory representation for parsing tables and messages themselves. -** Functions in this file are used by generated code and possibly reflection. -** -** The definitions in this file are internal to upb. -**/ - -#ifndef UPB_MESSAGE_INTERNAL_H_ -#define UPB_MESSAGE_INTERNAL_H_ - -#include -#include - - // Must be last. #ifdef __cplusplus @@ -2375,7 +2372,15 @@ typedef struct { } upb_Message_InternalData; typedef struct { - upb_Message_InternalData* internal; + // LINT.IfChange(internal_layout) + union { + upb_Message_InternalData* internal; + + // Force 8-byte alignment, since the data members may contain members that + // require 8-byte alignment. + double d; + }; + // LINT.ThenChange(//depot/google3/third_party/upb/upb/message/internal/map_entry.h:internal_layout) /* Message data follows. */ } upb_Message_Internal; @@ -2418,6 +2423,10 @@ bool _upb_Message_AddUnknown(upb_Message* msg, const char* data, size_t len, #endif /* UPB_MESSAGE_INTERNAL_H_ */ +#ifndef UPB_MESSAGE_INTERNAL_ACCESSORS_H_ +#define UPB_MESSAGE_INTERNAL_ACCESSORS_H_ + + // Must be last. #if defined(__GNUC__) && !defined(__clang__) @@ -2496,7 +2505,6 @@ UPB_INLINE uint32_t _upb_getoneofcase_field(const upb_Message* msg, } // LINT.ThenChange(GoogleInternalName2) -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/presence.ts:presence_logic) UPB_INLINE bool _upb_MiniTableField_InOneOf(const upb_MiniTableField* field) { return field->presence < 0; @@ -2521,8 +2529,6 @@ UPB_INLINE void _upb_Message_SetPresence(upb_Message* msg, } } -// LINT.IfChange(message_raw_fields) - UPB_INLINE bool _upb_MiniTable_ValueIsNonZero(const void* default_val, const upb_MiniTableField* field) { char zero[16] = {0}; @@ -2561,8 +2567,6 @@ UPB_INLINE void _upb_MiniTable_CopyFieldData(void* to, const void* from, UPB_UNREACHABLE(); } -// LINT.ThenChange(//depot/google3/third_party/upb/js/impl/upb_bits/message.ts:message_raw_fields) - UPB_INLINE size_t _upb_MiniTable_ElementSizeLg2(const upb_MiniTableField* field) { const unsigned char table[] = { @@ -2771,7 +2775,7 @@ UPB_INLINE upb_Map* _upb_Message_GetOrCreateMutableMap( #endif -#endif // UPB_MESSAGE_ACCESSORS_INTERNAL_H_ +#endif // UPB_MESSAGE_INTERNAL_ACCESSORS_H_ // Must be last. @@ -3357,9 +3361,9 @@ upb_MiniTable* upb_MiniTable_BuildWithBuf(const char* data, size_t len, // Must be last. struct upb_MiniTableFile { - const upb_MiniTable** msgs; - const upb_MiniTableEnum** enums; - const upb_MiniTableExtension** exts; + const struct upb_MiniTable** msgs; + const struct upb_MiniTableEnum** enums; + const struct upb_MiniTableExtension** exts; int msg_count; int enum_count; int ext_count; @@ -11307,8 +11311,8 @@ double _upb_NoLocaleStrtod(const char *str, char **endptr); #endif /* UPB_LEX_STRTOD_H_ */ -#ifndef UPB_MEM_ARENA_INTERNAL_H_ -#define UPB_MEM_ARENA_INTERNAL_H_ +#ifndef UPB_MEM_INTERNAL_ARENA_H_ +#define UPB_MEM_INTERNAL_ARENA_H_ // Must be last. @@ -11390,7 +11394,7 @@ UPB_INLINE bool upb_Arena_HasInitialBlock(upb_Arena* arena) { } -#endif /* UPB_MEM_ARENA_INTERNAL_H_ */ +#endif /* UPB_MEM_INTERNAL_ARENA_H_ */ #ifndef UPB_PORT_ATOMIC_H_ #define UPB_PORT_ATOMIC_H_ @@ -11486,8 +11490,8 @@ extern "C" { #define UPB_WIRE_READER_H_ -#ifndef UPB_WIRE_SWAP_INTERNAL_H_ -#define UPB_WIRE_SWAP_INTERNAL_H_ +#ifndef UPB_WIRE_INTERNAL_SWAP_H_ +#define UPB_WIRE_INTERNAL_SWAP_H_ // Must be last. @@ -11519,7 +11523,7 @@ UPB_INLINE uint64_t _upb_BigEndian_Swap64(uint64_t val) { #endif -#endif /* UPB_WIRE_SWAP_INTERNAL_H_ */ +#endif /* UPB_WIRE_INTERNAL_SWAP_H_ */ #ifndef UPB_WIRE_TYPES_H_ #define UPB_WIRE_TYPES_H_ @@ -12399,8 +12403,8 @@ upb_MethodDef* _upb_MethodDefs_New( #endif /* UPB_REFLECTION_METHOD_DEF_INTERNAL_H_ */ -#ifndef UPB_WIRE_COMMON_INTERNAL_H_ -#define UPB_WIRE_COMMON_INTERNAL_H_ +#ifndef UPB_WIRE_INTERNAL_COMMON_H_ +#define UPB_WIRE_INTERNAL_COMMON_H_ // Must be last. @@ -12419,15 +12423,15 @@ enum { }; -#endif /* UPB_WIRE_COMMON_INTERNAL_H_ */ +#endif /* UPB_WIRE_INTERNAL_COMMON_H_ */ /* * Internal implementation details of the decoder that are shared between * decode.c and decode_fast.c. */ -#ifndef UPB_WIRE_DECODE_INTERNAL_H_ -#define UPB_WIRE_DECODE_INTERNAL_H_ +#ifndef UPB_WIRE_INTERNAL_DECODE_H_ +#define UPB_WIRE_INTERNAL_DECODE_H_ #include "utf8_range.h" @@ -12550,7 +12554,7 @@ UPB_INLINE uint32_t _upb_FastDecoder_LoadTag(const char* ptr) { } -#endif /* UPB_WIRE_DECODE_INTERNAL_H_ */ +#endif /* UPB_WIRE_INTERNAL_DECODE_H_ */ #ifndef UPB_MINI_DESCRIPTOR_INTERNAL_BASE92_H_ #define UPB_MINI_DESCRIPTOR_INTERNAL_BASE92_H_ @@ -12746,6 +12750,8 @@ UPB_INLINE const char* upb_MdDecoder_DecodeBase92Varint( #undef UPB_POISON_MEMORY_REGION #undef UPB_UNPOISON_MEMORY_REGION #undef UPB_ASAN +#undef UPB_ASAN_GUARD_SIZE +#undef UPB_CLANG_ASAN #undef UPB_TREAT_PROTO2_ENUMS_LIKE_PROTO3 #undef UPB_DEPRECATED #undef UPB_GNUC_MIN