Merge remote-tracking branch 'upstream/main' into fix_14509

pull/14538/head
Jason Lunn 1 year ago
commit a3f9d0bd4f
  1. 2
      .github/workflows/test_upb.yml
  2. 1
      python/dist/BUILD.bazel
  3. 2
      python/google/protobuf/internal/api_implementation.py
  4. 3
      src/google/protobuf/extension_set.cc
  5. 16
      src/google/protobuf/extension_set.h
  6. 5
      src/google/protobuf/generated_message_reflection.cc
  7. 217
      upb/BUILD
  8. 1
      upb/cmake/BUILD.bazel
  9. 192
      upb/reflection/BUILD
  10. 22
      upb_generator/BUILD
  11. 8
      upb_generator/protoc-gen-upbdefs.cc

@ -76,7 +76,7 @@ jobs:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-windows"
bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/... //protos/... //protos_generator/...
exclude-targets: -//python:conformance_test -//upb:def_builder_test
exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test
macos:
strategy:

@ -246,6 +246,7 @@ pkg_files(
"//upb/mini_descriptor:source_files",
"//upb/mini_table:source_files",
"//upb/port:source_files",
"//upb/reflection:source_files",
"//upb/text:source_files",
"//upb/util:source_files",
"//upb/wire:source_files",

@ -13,6 +13,8 @@ import os
import sys
import warnings
_GOOGLE3_PYTHON_UPB_DEFAULT = False
def _ApiVersionToImplementationType(api_version):
if api_version == 2:

@ -11,6 +11,7 @@
#include "google/protobuf/extension_set.h"
#include <atomic>
#include <string>
#include <tuple>
#include <type_traits>
@ -1933,6 +1934,8 @@ LazyEagerVerifyFnType FindExtensionLazyEagerVerifyFn(
return nullptr;
}
std::atomic<ExtensionSet::LazyMessageExtension* (*)(Arena* arena)>
ExtensionSet::maybe_create_lazy_extension_;
} // namespace internal
} // namespace protobuf

@ -16,6 +16,7 @@
#define GOOGLE_PROTOBUF_EXTENSION_SET_H__
#include <algorithm>
#include <atomic>
#include <cassert>
#include <string>
#include <type_traits>
@ -57,6 +58,7 @@ class FeatureSet;
namespace internal {
class FieldSkipper; // wire_format_lite.h
class WireFormat;
void InitializeLazyExtensionSet();
} // namespace internal
} // namespace protobuf
} // namespace google
@ -508,6 +510,8 @@ class PROTOBUF_EXPORT ExtensionSet {
friend class google::protobuf::Reflection;
friend class google::protobuf::internal::WireFormat;
friend void internal::InitializeLazyExtensionSet();
const int32_t& GetRefInt32(int number, const int32_t& default_value) const;
const int64_t& GetRefInt64(int number, const int64_t& default_value) const;
const uint32_t& GetRefUInt32(int number, const uint32_t& default_value) const;
@ -579,7 +583,13 @@ class PROTOBUF_EXPORT ExtensionSet {
virtual void UnusedKeyMethod(); // Dummy key method to avoid weak vtable.
};
// Give access to function defined below to see LazyMessageExtension.
friend LazyMessageExtension* MaybeCreateLazyExtension(Arena* arena);
static LazyMessageExtension* MaybeCreateLazyExtensionImpl(Arena* arena);
static LazyMessageExtension* MaybeCreateLazyExtension(Arena* arena) {
auto* f = maybe_create_lazy_extension_.load(std::memory_order_relaxed);
return f != nullptr ? f(arena) : nullptr;
}
static std::atomic<LazyMessageExtension* (*)(Arena* arena)>
maybe_create_lazy_extension_;
struct Extension {
// The order of these fields packs Extension into 24 bytes when using 8
// byte alignment. Consider this when adding or removing fields here.
@ -1529,10 +1539,6 @@ class ExtensionIdentifier {
// Generated accessors
// Used to retrieve a lazy extension, may return nullptr in some environments.
extern PROTOBUF_ATTRIBUTE_WEAK ExtensionSet::LazyMessageExtension*
MaybeCreateLazyExtension(Arena* arena);
// Define a specialization of ExtensionIdentifier for bootstrapped extensions
// that we need to register lazily.
template <>

@ -16,6 +16,7 @@
#include <cstdint>
#include <cstring>
#include <string>
#include <type_traits>
#include "absl/base/call_once.h"
#include "absl/base/casts.h"
@ -93,6 +94,9 @@ void InitializeFileDescriptorDefaultInstances() {
#endif // !defined(PROTOBUF_CONSTINIT_DEFAULT_INSTANCES)
}
void InitializeLazyExtensionSet() {
}
bool ParseNamedEnum(const EnumDescriptor* descriptor, absl::string_view name,
int* value) {
const EnumValueDescriptor* d = descriptor->FindValueByName(name);
@ -3660,6 +3664,7 @@ void AddDescriptorsImpl(const DescriptorTable* table) {
// Reflection refers to the default fields so make sure they are initialized.
internal::InitProtobufDefaults();
internal::InitializeFileDescriptorDefaultInstances();
internal::InitializeLazyExtensionSet();
// Ensure all dependent descriptors are registered to the generated descriptor
// pool and message factory.

@ -8,14 +8,7 @@
load("@rules_python//python:defs.bzl", "py_binary")
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS")
load("//bazel:upb_minitable_proto_library.bzl", "upb_minitable_proto_library")
load("//bazel:upb_proto_library.bzl", "upb_proto_reflection_library")
load("//bazel:upb_proto_library_internal/copts.bzl", "upb_proto_library_copts")
load(
"//upb_generator:bootstrap_compiler.bzl",
"bootstrap_cc_library",
"bootstrap_upb_proto_library",
)
# begin:google_only
# load("//tools/build_defs/kotlin/native:rules.bzl", "kt_native_interop_hint")
@ -104,14 +97,14 @@ cc_library(
# implementation, but depends on :upb and exposes a few more hdrs.
#
# This is public only because we have no way of visibility-limiting it to
# upb_proto_library() only. This interface is not stable and by using it you
# upb_c_proto_library() only. This interface is not stable and by using it you
# give up any backward compatibility guarantees.
cc_library(
name = "generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
hdrs = ["generated_code_support.h"],
copts = UPB_DEFAULT_COPTS,
textual_hdrs = [
"//upb/port:inc",
":port_inc",
],
visibility = ["//visibility:public"],
deps = [
@ -133,176 +126,54 @@ cc_library(
name = "generated_cpp_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
copts = UPB_DEFAULT_COPTS,
textual_hdrs = [
"//upb/port:inc",
":port_inc",
],
visibility = ["//visibility:public"],
)
cc_library(
name = "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
hdrs = [
"reflection/def.h",
"reflection/internal/def_pool.h",
],
copts = UPB_DEFAULT_COPTS,
textual_hdrs = [
"//upb/port:inc",
],
visibility = ["//visibility:public"],
deps = [
":mem",
":mini_descriptor",
":reflection_internal",
],
)
# Aliases ######################################################################
# TODO: Remove these.
bootstrap_upb_proto_library(
name = "descriptor_upb_proto",
base_dir = "reflection/",
# TODO: Export 'net/proto2/proto/descriptor.upb.h' and remove "-layering_check".
features = ["-layering_check"],
google3_src_files = ["net/proto2/proto/descriptor.proto"],
google3_src_rules = ["//net/proto2/proto:descriptor_proto_source"],
oss_src_files = ["google/protobuf/descriptor.proto"],
oss_src_rules = ["//:descriptor_proto_srcs"],
oss_strip_prefix = "third_party/protobuf/github/bootstrap/src",
proto_lib_deps = ["//:descriptor_proto"],
alias(
name = "base",
actual = "//upb/base",
visibility = ["//visibility:public"],
)
upb_proto_reflection_library(
name = "descriptor_upb_proto_reflection",
alias(
name = "base_internal",
actual = "//upb/base:internal",
visibility = ["//visibility:public"],
deps = ["//:descriptor_proto"],
)
upb_minitable_proto_library(
name = "descriptor_upb_minitable_proto",
visibility = [
"//upb:__subpackages__",
# begin:github_only
"//python:__subpackages__",
# end:github_only
],
deps = ["//:descriptor_proto"],
)
# TODO: Once we can delete the deprecated forwarding headers
# (= everything in upb/) we can move this build target down into reflection/
bootstrap_cc_library(
name = "reflection",
hdrs = [
"reflection/def.h",
"reflection/def.hpp",
"reflection/message.h",
"reflection/message.hpp",
],
bootstrap_deps = [":reflection_internal"],
copts = UPB_DEFAULT_COPTS,
alias(
name = "collections",
actual = "//upb/collections",
deprecation = "use upb:message instead",
visibility = ["//visibility:public"],
deps = [
":base",
":mem",
":message",
":message_types",
":message_value",
":port",
],
)
bootstrap_cc_library(
name = "reflection_internal",
srcs = [
"reflection/def_pool.c",
"reflection/def_type.c",
"reflection/desc_state.c",
"reflection/enum_def.c",
"reflection/enum_reserved_range.c",
"reflection/enum_value_def.c",
"reflection/extension_range.c",
"reflection/field_def.c",
"reflection/file_def.c",
"reflection/internal/def_builder.c",
"reflection/internal/def_builder.h",
"reflection/internal/strdup2.c",
"reflection/internal/strdup2.h",
"reflection/message.c",
"reflection/message_def.c",
"reflection/message_reserved_range.c",
"reflection/method_def.c",
"reflection/oneof_def.c",
"reflection/service_def.c",
],
hdrs = [
"reflection/common.h",
"reflection/def.h",
"reflection/def.hpp",
"reflection/def_pool.h",
"reflection/def_type.h",
"reflection/enum_def.h",
"reflection/enum_reserved_range.h",
"reflection/enum_value_def.h",
"reflection/extension_range.h",
"reflection/field_def.h",
"reflection/file_def.h",
"reflection/internal/def_pool.h",
"reflection/internal/desc_state.h",
"reflection/internal/enum_def.h",
"reflection/internal/enum_reserved_range.h",
"reflection/internal/enum_value_def.h",
"reflection/internal/extension_range.h",
"reflection/internal/field_def.h",
"reflection/internal/file_def.h",
"reflection/internal/message_def.h",
"reflection/internal/message_reserved_range.h",
"reflection/internal/method_def.h",
"reflection/internal/oneof_def.h",
"reflection/internal/service_def.h",
"reflection/message.h",
"reflection/message.hpp",
"reflection/message_def.h",
"reflection/message_reserved_range.h",
"reflection/method_def.h",
"reflection/oneof_def.h",
"reflection/service_def.h",
],
bootstrap_deps = [":descriptor_upb_proto"],
copts = UPB_DEFAULT_COPTS,
alias(
name = "descriptor_upb_proto",
actual = "//upb/reflection:descriptor_upb_proto",
visibility = ["//visibility:public"],
deps = [
":base",
":hash",
":mem",
":message",
":message_accessors",
":message_types",
":message_value",
":mini_descriptor",
":mini_descriptor_internal",
":mini_table",
":port",
],
)
# Aliases ######################################################################
# TODO: Remove these.
alias(
name = "base",
actual = "//upb/base",
name = "descriptor_upb_minitable_proto",
actual = "//upb/reflection:descriptor_upb_minitable_proto",
visibility = ["//visibility:public"],
)
alias(
name = "base_internal",
actual = "//upb/base:internal",
name = "descriptor_upb_proto_reflection",
actual = "//upb/reflection:descriptor_upb_proto_reflection",
visibility = ["//visibility:public"],
)
alias(
name = "collections",
actual = "//upb/collections",
deprecation = "use upb:message instead",
name = "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
actual = "//upb/reflection:generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
visibility = ["//visibility:public"],
)
@ -438,6 +309,24 @@ alias(
visibility = ["//visibility:public"],
)
alias(
name = "port_inc",
actual = "//upb/port:inc",
visibility = ["//visibility:public"],
)
alias(
name = "reflection",
actual = "//upb/reflection",
visibility = ["//visibility:public"],
)
alias(
name = "reflection_internal",
actual = "//upb/reflection:internal",
visibility = ["//visibility:public"],
)
alias(
name = "text",
actual = "//upb/text",
@ -474,28 +363,6 @@ alias(
visibility = ["//visibility:public"],
)
# Tests ########################################################################
cc_test(
name = "def_builder_test",
srcs = [
"reflection/common.h",
"reflection/def_type.h",
"reflection/internal/def_builder.h",
"reflection/internal/def_builder_test.cc",
],
deps = [
":descriptor_upb_proto",
":hash",
":mem",
":port",
":reflection",
":reflection_internal",
"@com_google_googletest//:gtest_main",
"@com_google_absl//absl/strings",
],
)
# Internal C/C++ libraries #####################################################
cc_binary(

@ -96,6 +96,7 @@ sh_test(
"//upb/mini_descriptor:source_files",
"//upb/mini_table:source_files",
"//upb/port:source_files",
"//upb/reflection:source_files",
"//upb/text:source_files",
"//upb/wire:source_files",
"@utf8_range//:utf8_range_srcs",

@ -0,0 +1,192 @@
# Copyright (c) 2009-2021, Google LLC
# All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file or at
# https://developers.google.com/open-source/licenses/bsd
load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS")
load("//bazel:upb_minitable_proto_library.bzl", "upb_minitable_proto_library")
load("//bazel:upb_proto_library.bzl", "upb_proto_reflection_library")
load(
"//upb_generator:bootstrap_compiler.bzl",
"bootstrap_cc_library",
"bootstrap_upb_proto_library",
)
bootstrap_upb_proto_library(
name = "descriptor_upb_proto",
base_dir = "",
# TODO: Export 'net/proto2/proto/descriptor.upb.h' and remove "-layering_check".
features = ["-layering_check"],
google3_src_files = ["net/proto2/proto/descriptor.proto"],
google3_src_rules = ["//net/proto2/proto:descriptor_proto_source"],
oss_src_files = ["google/protobuf/descriptor.proto"],
oss_src_rules = ["//:descriptor_proto_srcs"],
oss_strip_prefix = "third_party/protobuf/github/bootstrap/src",
proto_lib_deps = ["//:descriptor_proto"],
visibility = ["//visibility:public"],
)
upb_proto_reflection_library(
name = "descriptor_upb_proto_reflection",
visibility = ["//visibility:public"],
deps = ["//:descriptor_proto"],
)
upb_minitable_proto_library(
name = "descriptor_upb_minitable_proto",
visibility = ["//visibility:public"],
deps = ["//:descriptor_proto"],
)
bootstrap_cc_library(
name = "reflection",
hdrs = [
"def.h",
"def.hpp",
"message.h",
"message.hpp",
],
bootstrap_deps = [":internal"],
copts = UPB_DEFAULT_COPTS,
visibility = ["//visibility:public"],
deps = [
"//upb:base",
"//upb:mem",
"//upb:message",
"//upb:message_types",
"//upb:message_value",
"//upb:port",
],
)
bootstrap_cc_library(
name = "internal",
srcs = [
"def_pool.c",
"def_type.c",
"desc_state.c",
"enum_def.c",
"enum_reserved_range.c",
"enum_value_def.c",
"extension_range.c",
"field_def.c",
"file_def.c",
"internal/def_builder.c",
"internal/def_builder.h",
"internal/strdup2.c",
"internal/strdup2.h",
"message.c",
"message_def.c",
"message_reserved_range.c",
"method_def.c",
"oneof_def.c",
"service_def.c",
],
hdrs = [
"common.h",
"def.h",
"def.hpp",
"def_pool.h",
"def_type.h",
"enum_def.h",
"enum_reserved_range.h",
"enum_value_def.h",
"extension_range.h",
"field_def.h",
"file_def.h",
"internal/def_pool.h",
"internal/desc_state.h",
"internal/enum_def.h",
"internal/enum_reserved_range.h",
"internal/enum_value_def.h",
"internal/extension_range.h",
"internal/field_def.h",
"internal/file_def.h",
"internal/message_def.h",
"internal/message_reserved_range.h",
"internal/method_def.h",
"internal/oneof_def.h",
"internal/service_def.h",
"message.h",
"message.hpp",
"message_def.h",
"message_reserved_range.h",
"method_def.h",
"oneof_def.h",
"service_def.h",
],
bootstrap_deps = [":descriptor_upb_proto"],
copts = UPB_DEFAULT_COPTS,
visibility = ["//visibility:public"],
deps = [
"//upb:base",
"//upb:hash",
"//upb:mem",
"//upb:message",
"//upb:message_accessors",
"//upb:message_types",
"//upb:message_value",
"//upb:mini_descriptor",
"//upb:mini_descriptor_internal",
"//upb:mini_table",
"//upb:port",
],
)
cc_library(
name = "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
hdrs = [
"def.h",
"internal/def_pool.h",
],
copts = UPB_DEFAULT_COPTS,
textual_hdrs = [
"//upb:port_inc",
],
visibility = ["//visibility:public"],
deps = [
"//upb:mem",
"//upb:mini_descriptor",
"//upb:reflection_internal",
],
)
cc_test(
name = "def_builder_test",
srcs = [
"common.h",
"def_type.h",
"internal/def_builder.h",
"internal/def_builder_test.cc",
],
deps = [
":descriptor_upb_proto",
"@com_google_googletest//:gtest_main",
"//upb:hash",
"//upb:mem",
"//upb:port",
"//upb:reflection",
"//upb:reflection_internal",
"@com_google_absl//absl/strings",
],
)
# begin:github_only
filegroup(
name = "source_files",
srcs = glob(
[
"**/*.c",
"**/*.h",
"**/*.hpp",
],
exclude = ["stage0/**"],
),
visibility = [
"//upb/cmake:__pkg__",
"//python/dist:__pkg__",
]
)
# end:github_only

@ -79,7 +79,7 @@ bootstrap_upb_proto_library(
oss_strip_prefix = "third_party/protobuf/github/bootstrap/src",
proto_lib_deps = ["//:compiler_plugin_proto"],
visibility = ["//upb:friends"],
deps = ["//upb:descriptor_upb_proto"],
deps = ["//upb/reflection:descriptor_upb_proto"],
)
upb_proto_reflection_library(
@ -97,7 +97,7 @@ bootstrap_cc_library(
"common.h",
],
bootstrap_deps = [
"//upb:reflection",
"//upb/reflection:reflection",
],
copts = UPB_DEFAULT_CPPOPTS,
visibility = ["//protos_generator:__pkg__"],
@ -119,8 +119,8 @@ bootstrap_cc_library(
],
bootstrap_deps = [
":common",
"//upb:reflection",
"//upb:descriptor_upb_proto",
"//upb/reflection:reflection",
"//upb/reflection:descriptor_upb_proto",
],
copts = UPB_DEFAULT_CPPOPTS,
visibility = ["//visibility:public"],
@ -154,8 +154,8 @@ bootstrap_cc_library(
],
bootstrap_deps = [
":plugin_upb_proto",
"//upb:descriptor_upb_proto",
"//upb:reflection",
"//upb/reflection:descriptor_upb_proto",
"//upb/reflection:reflection",
],
copts = UPB_DEFAULT_CPPOPTS,
visibility = ["//protos_generator:__pkg__"],
@ -177,7 +177,7 @@ bootstrap_cc_library(
"names.h",
],
bootstrap_deps = [
"//upb:reflection",
"//upb/reflection:reflection",
],
copts = UPB_DEFAULT_CPPOPTS,
visibility = ["//protos_generator:__pkg__"],
@ -244,8 +244,8 @@ bootstrap_cc_binary(
":names",
":plugin",
":plugin_upb_proto",
"//upb:descriptor_upb_proto",
"//upb:reflection",
"//upb/reflection:descriptor_upb_proto",
"//upb/reflection:reflection",
],
copts = UPB_DEFAULT_CPPOPTS,
visibility = ["//visibility:public"],
@ -283,8 +283,8 @@ bootstrap_cc_binary(
":names",
":plugin",
":plugin_upb_proto",
"//upb:descriptor_upb_proto",
"//upb:reflection",
"//upb/reflection:descriptor_upb_proto",
"//upb/reflection:reflection",
],
copts = UPB_DEFAULT_CPPOPTS,
visibility = ["//visibility:public"],

@ -70,17 +70,13 @@ void WriteDefHeader(upb::FileDefPtr file, Output& output) {
"#define $0_UPBDEFS_H_\n\n"
"#include \"upb/reflection/def.h\"\n"
"#include \"upb/reflection/internal/def_pool.h\"\n"
"#include \"upb/port/def.inc\"\n"
"\n"
"#include \"upb/port/def.inc\" // Must be last.\n"
"#ifdef __cplusplus\n"
"extern \"C\" {\n"
"#endif\n\n",
ToPreproc(file.name()));
output("#include \"upb/reflection/def.h\"\n");
output("\n");
output("#include \"upb/port/def.inc\"\n");
output("\n");
output("extern _upb_DefPool_Init $0;\n", DefInitSymbol(file));
output("\n");

Loading…
Cancel
Save