diff --git a/BUILD.bazel b/BUILD.bazel index 335261439c..cc623aac75 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -801,10 +801,13 @@ cc_test( "src/google/protobuf/**/*", # Files for csharp_bootstrap_unittest.cc. "conformance/**/*", - ]) + glob([ - # Files for csharp_bootstrap_unittest.cc. - "csharp/src/**/*", - ], allow_empty=True), + ]) + glob( + [ + # Files for csharp_bootstrap_unittest.cc. + "csharp/src/**/*", + ], + allow_empty = True, + ), includes = [ "src/", ], @@ -1020,10 +1023,6 @@ py_proto_library( py_library( name = "protobuf_python", - deps = [ - ":well_known_types_py_pb2", - ":python_srcs", - ], data = select({ "//conditions:default": [], ":use_fast_cpp_protos": [ @@ -1031,6 +1030,10 @@ py_library( ":python/google/protobuf/pyext/_message.so", ], }), + deps = [ + ":python_srcs", + ":well_known_types_py_pb2", + ], ) # Copy the test proto files from src/google/protobuf to @@ -1058,8 +1061,8 @@ py_proto_library( default_runtime = "", protoc = ":protoc", srcs_version = "PY2AND3", - deps = [":well_known_types_py_pb2"], visibility = ["//visibility:public"], + deps = [":well_known_types_py_pb2"], ) py_proto_library( @@ -1072,8 +1075,8 @@ py_proto_library( default_runtime = ":protobuf_python", protoc = ":protoc", srcs_version = "PY2AND3", - deps = [":python_common_test_protos"], visibility = ["//visibility:public"], + deps = [":python_common_test_protos"], ) py_library( @@ -1139,12 +1142,14 @@ proto_lang_toolchain( alias( name = "objectivec", actual = "//objectivec", + tags = ["manual"], visibility = ["//visibility:public"], ) alias( name = "protobuf_objc", actual = "//objectivec", + tags = ["manual"], visibility = ["//visibility:public"], ) @@ -1155,7 +1160,7 @@ alias( genrule( name = "generated_protos", srcs = ["src/google/protobuf/unittest_import.proto"], - outs = ["unittest_gen.proto"], + outs = ["unittest_gen_import.proto"], cmd = "cat $(SRCS) | sed 's|google/|src/google/|' > $(OUTS)", ) @@ -1474,28 +1479,31 @@ internal_gen_kt_protos( # Files included in all source distributions pkg_files( name = "common_dist_files", - srcs = glob([ - "*.bzl", - "cmake/*.cmake", - "cmake/*.in", - "editors/*", - - # Several of these files are generated by autogen.sh, so using - # glob() lets us ignore them if they are missing. (This is not good - # practice, though.) - "Makefile.in", - "aclocal.m4", - "ar-lib", - "compile", - "config*", - "depcomp", - "install-sh", - "ltmain.sh", - "m4/*.m4", - "missing", - "protobuf*.pc.in", - "test-driver", - ], allow_empty = True) + [ + srcs = glob( + [ + "*.bzl", + "cmake/*.cmake", + "cmake/*.in", + "editors/*", + + # Several of these files are generated by autogen.sh, so using + # glob() lets us ignore them if they are missing. (This is not good + # practice, though.) + "Makefile.in", + "aclocal.m4", + "ar-lib", + "compile", + "config*", + "depcomp", + "install-sh", + "ltmain.sh", + "m4/*.m4", + "missing", + "protobuf*.pc.in", + "test-driver", + ], + allow_empty = True, + ) + [ "BUILD.bazel", "CHANGES.txt", "CMakeLists.txt", @@ -1569,15 +1577,6 @@ pkg_files( visibility = ["//pkg:__pkg__"], ) -# Additional files for PHP -pkg_files( - name = "php_dist_files", - srcs = [ - "composer.json", - ], - visibility = ["//pkg:__pkg__"], -) - # Python runtime pkg_files( name = "python_dist_files", diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 60be807190..e36a38e7f1 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -121,6 +121,7 @@ java_export( resources = [ "//:lite_well_known_protos", ], + tags = ["manual"], runtime_deps = [":lite"], ) @@ -156,6 +157,7 @@ java_export( resources = [ "//:well_known_protos", ], + tags = ["manual"], runtime_deps = [":core"], ) diff --git a/java/kotlin-lite/BUILD.bazel b/java/kotlin-lite/BUILD.bazel index 47b02c2ae4..09f0decff2 100644 --- a/java/kotlin-lite/BUILD.bazel +++ b/java/kotlin-lite/BUILD.bazel @@ -37,6 +37,7 @@ kt_jvm_export( maven_coordinates = "com.google.protobuf:protobuf-kotlin-lite:%s" % PROTOBUF_JAVA_VERSION, pom_template = "//java/kotlin-lite:pom_template.xml", resources = ["//:well_known_protos"], + tags = ["manual"], runtime_deps = [ ":lite_extensions", ":well_known_protos_kotlin_lite", diff --git a/java/kotlin/BUILD.bazel b/java/kotlin/BUILD.bazel index a3cf83d46b..859b86ba3f 100644 --- a/java/kotlin/BUILD.bazel +++ b/java/kotlin/BUILD.bazel @@ -59,6 +59,7 @@ kt_jvm_export( maven_coordinates = "com.google.protobuf:protobuf-kotlin:%s" % PROTOBUF_JAVA_VERSION, pom_template = "//java/kotlin:pom_template.xml", resources = ["//:well_known_protos"], + tags = ["manual"], runtime_deps = [ ":bytestring_lib", ":full_extensions", diff --git a/java/util/BUILD.bazel b/java/util/BUILD.bazel index bea3099daa..e3804fb978 100644 --- a/java/util/BUILD.bazel +++ b/java/util/BUILD.bazel @@ -27,6 +27,7 @@ java_export( deploy_env = ["//java/core"], maven_coordinates = "com.google.protobuf:protobuf-java-util:%s" % PROTOBUF_JAVA_VERSION, pom_template = "pom_template.xml", + tags = ["manual"], visibility = ["//java:__pkg__"], runtime_deps = [":util"], ) diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index c5ed05e0f5..a197c74ec7 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -211,7 +211,6 @@ pkg_filegroup( name = "php_srcs", srcs = [ ":dist_common", - "//:php_dist_files", "//php:dist_files", ], ) @@ -339,11 +338,9 @@ gen_automake_file_lists( "//csharp:dist_files": "dist_csharp2", "//:objectivec_dist_files": "dist_objectivec", "//objectivec:dist_files": "dist_objectivec2", - "//:php_dist_files": "dist_php", - "//php:dist_files": "dist_php2", + "//php:dist_files": "dist_php", "//:python_dist_files": "dist_python", "//ruby:dist_files": "dist_ruby", - "//js:dist_files": "dist_js", }, ) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9e664546ed..eab12e33cb 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -114,6 +114,6 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "2abcb7dd8a524d73322082951d334392c76721b9", - sha256 = "f583f1389a7531940a51221b21b69a421b4c12811a91fb5df493cd0bb13ee5d6", + commit = "0e8772fc20e5a0a2fa1f326c79d494374871ef94", + sha256 = "2a7ccd5a8e20c5b2f6efafb305f734c7ad1cc5615706f7a35cd4fbad436d5101", ) diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 8da8329beb..13955260a7 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,2 +1,6 @@ PROTOC_VERSION = '3.20.1' PROTOBUF_JAVA_VERSION = '3.20.1' +PROTOBUF_PYTHON_VERSION = '3.20.1' + +# Remove once upb has been updated to use PROTOBUF_PYTHON_VERSION instead. +PROTOBUF_VERSION = PROTOBUF_PYTHON_VERSION diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index c3436d4119..3b5f16c389 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -414,6 +414,8 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final { // Provides access to protected GetOwningArena to generated messages. static Arena* GetOwningArena(const T* p) { return p->GetOwningArena(); } + static void InternalSwap(T* a, T* b) { a->InternalSwap(b); } + static Arena* GetArenaForAllocationInternal( const T* p, std::true_type /*is_derived_from*/) { return p->GetArenaForAllocation(); diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index b29962fd83..4b7c5c9d91 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -202,64 +202,6 @@ inline VerifySimpleType VerifyInt32TypeToVerifyCustom(VerifyInt32Type t) { static_cast(VerifyInt32Type::kCustom)]; } -// Returns one of int32 verification types: kNever, kCustom, kAlways. -// -// We need to verify int32 encoding to detect non-canonical encoding (5B for -// negative int32) and fallback to eager parsing. -// -// kNever skips int32 check because there is no int32 field. kAlways -// unconditionally verifies int32 encoding because all or almost varint fields -// are int32. Otherwise, kCustom verifies int32 encoding only on exact field -// number match. Note the following tweaks: -// --uint32 very likely causes false positives. Having one requires kCustom. -// --kCustom may be cheap enough if all int32 fields fit into a bitmask. -// --Otherwise, try always check if X% of varint fields are int32. -VerifyInt32Type ShouldVerifyInt32(const Descriptor* descriptor) { - int num_int32 = 0; - int num_int32_big_number = 0; - int num_uint32 = 0; - int num_other_varint = 0; - - for (const auto* field : FieldRange(descriptor)) { - switch (field->type()) { - case FieldDescriptor::TYPE_INT32: - ++num_int32; - if (field->number() > 64) ++num_int32_big_number; - break; - case FieldDescriptor::TYPE_UINT32: - ++num_uint32; - ++num_other_varint; - break; - default: - if (internal::WireFormat::WireTypeForFieldType(field->type()) == - internal::WireFormatLite::WIRETYPE_VARINT) { - ++num_other_varint; - } - break; - } - } - - // If there is no int32 fields, no need to check int32 encoding. - if (num_int32 == 0) return VerifyInt32Type::kNever; - - // If all varint fields are int32, *always* check int32 encoding. - if (num_other_varint == 0) return VerifyInt32Type::kAlways; - - // Negative uint32 encoding will cause fallback eager parsing as it appears - // non-canonical encoding. Also, if all int32 fields fit into a 64 bit mask, - // checking bitmask is affordable. Try exact match in these cases. - if (num_uint32 > 0 || num_int32_big_number == 0) { - return VerifyInt32Type::kCustom; - } - - // If a given varint is likely int32, we should just always check. Let's use - // an arbitrary threshold of 75% (#int32 / #varints). - constexpr int kLikelyInt32Pct = 75; - return (100 * num_int32) / (num_int32 + num_other_varint) >= kLikelyInt32Pct - ? VerifyInt32Type::kAlways - : VerifyInt32Type::kCustom; -} - } // namespace bool IsLazy(const FieldDescriptor* field, const Options& options, @@ -574,7 +516,7 @@ std::string FieldMemberName(const FieldDescriptor* field, bool split) { if (field->real_containing_oneof() == nullptr) { return StrCat(prefix, split_prefix, FieldName(field), "_"); } - // Oneof fields are enver split. + // Oneof fields are never split. GOOGLE_CHECK(!split); return StrCat(prefix, field->containing_oneof()->name(), "_.", FieldName(field), "_"); diff --git a/src/google/protobuf/compiler/cpp/options.h b/src/google/protobuf/compiler/cpp/options.h index 148866ff6e..5d935e9e6e 100644 --- a/src/google/protobuf/compiler/cpp/options.h +++ b/src/google/protobuf/compiler/cpp/options.h @@ -80,7 +80,7 @@ struct Options { bool annotate_accessor = false; bool unused_field_stripping = false; bool unverified_lazy_message_sets = false; - bool unverified_lazy = true; + bool unverified_lazy = false; bool profile_driven_inline_string = true; bool message_owned_arena_trial = false; bool force_split = false; diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 53bfd2d233..008c192253 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -1363,11 +1363,8 @@ class Map { } template void insert(InputIt first, InputIt last) { - for (InputIt it = first; it != last; ++it) { - iterator exist_it = find(it->first); - if (exist_it == end()) { - operator[](it->first) = it->second; - } + for (; first != last; ++first) { + try_emplace(first->first, first->second); } } void insert(std::initializer_list values) { diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index d2ef2d96d4..fa2bbbf599 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -285,6 +285,58 @@ TEST(MESSAGE_TEST_NAME, MergeFromUninitialized) { EXPECT_TRUE(TestUtil::EqualsToSerialized(q, p.SerializePartialAsString())); } +TEST(MESSAGE_TEST_NAME, ExplicitLazyExceedRecursionLimit) { + UNITTEST::NestedTestAllTypes original, parsed; + // Build proto with recursion depth of 3. + original.mutable_lazy_child() + ->mutable_child() + ->mutable_payload() + ->set_optional_int32(-1); + std::string serialized; + EXPECT_TRUE(original.SerializeToString(&serialized)); + + // User annotated LazyField ([lazy = true]) is eagerly verified and should + // catch the recursion limit violation. + io::ArrayInputStream array_stream(serialized.data(), serialized.size()); + io::CodedInputStream input_stream(&array_stream); + input_stream.SetRecursionLimit(2); + EXPECT_FALSE(parsed.ParseFromCodedStream(&input_stream)); + + // Lazy read results in parsing error which can be verified by not having + // expected value. + EXPECT_NE(parsed.lazy_child().child().payload().optional_int32(), -1); +} + +TEST(MESSAGE_TEST_NAME, NestedExplicitLazyExceedRecursionLimit) { + UNITTEST::NestedTestAllTypes original, parsed; + // Build proto with recursion depth of 5, with nested annotated LazyField. + original.mutable_lazy_child() + ->mutable_child() + ->mutable_lazy_child() + ->mutable_child() + ->mutable_payload() + ->set_optional_int32(-1); + std::string serialized; + EXPECT_TRUE(original.SerializeToString(&serialized)); + + // User annotated LazyField ([lazy = true]) is eagerly verified and should + // catch the recursion limit violation. + io::ArrayInputStream array_stream(serialized.data(), serialized.size()); + io::CodedInputStream input_stream(&array_stream); + input_stream.SetRecursionLimit(4); + EXPECT_FALSE(parsed.ParseFromCodedStream(&input_stream)); + + // Lazy read results in parsing error which can be verified by not having + // expected value. + EXPECT_NE(parsed.lazy_child() + .child() + .lazy_child() + .child() + .payload() + .optional_int32(), + -1); +} + TEST(MESSAGE_TEST_NAME, ParseFailsIfSubmessageTruncated) { UNITTEST::NestedTestAllTypes o, p; constexpr int kDepth = 5; diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index 315c775e29..dcb0ff758f 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -655,7 +655,7 @@ #define PROTOBUF_CONSTEXPR constexpr #else #define PROTOBUF_CONSTINIT -#define PROTOBUF_CONSTEXPR inline +#define PROTOBUF_CONSTEXPR #endif // Some globals with an empty non-trivial destructor are annotated with diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index d82e135ae7..3fb734e5cb 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -346,7 +346,7 @@ class RepeatedField final { int total_size_; // Pad the Rep after arena allow for power-of-two byte sizes when // sizeof(Element) > sizeof(Arena*). eg for 16-byte objects. - static constexpr size_t kRepHeaderSize = + static PROTOBUF_CONSTEXPR const size_t kRepHeaderSize = sizeof(Arena*) < sizeof(Element) ? sizeof(Element) : sizeof(Arena*); struct Rep { Arena* arena; diff --git a/version.json b/version.json index 5cebd3ddd2..e50d633261 100644 --- a/version.json +++ b/version.json @@ -10,7 +10,7 @@ "javascript": "3.21-dev", "objectivec": "3.21-dev", "php": "3.21-dev", - "python": "3.21-dev", + "python": "4.21-dev", "ruby": "3.21-dev" } }