diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 416ecc3cc8..bc706d9d63 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -31,22 +31,22 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:8-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 # TODO: b/318555165 - enable the layering check. Currently it does # not work correctly with the toolchain in this Docker image. - targets: //java/... //java/internal:java_version --features=-layering_check + targets: //java/... //java/internal:java_version //compatibility/... --features=-layering_check presubmit: false - name: OpenJDK 11 version: '11' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:11-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 - targets: //java/... //java/internal:java_version + targets: //java/... //java/internal:java_version //compatibility/... presubmit: false - name: OpenJDK 17 version: '17' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:17-1fdbb997433cb22c1e49ef75ad374a8d6bb88702 - targets: //java/... //java/internal:java_version + targets: //java/... //java/internal:java_version //compatibility/... presubmit: true - name: aarch64 version: 'aarch64' image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17 - targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test + targets: //java/... //compatibility/... //src/google/protobuf/compiler:protoc_aarch64_test presubmit: true name: ${{ !matrix.presubmit && inputs.continuous-prefix || '' }} Linux ${{ matrix.name }} diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 4ac2fc844a..3016ab345d 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -53,12 +53,13 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION --test_env=BAZEL=true ${{ matrix.ffi == 'FFI' && '--//ruby:ffi=enabled --test_env=PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI' || '' }} - - name: Archive log artifacts - if: ${{ matrix.presubmit || inputs.test-type == 'continuous' }} - uses: actions/upload-artifact@v4 - with: - name: test-logs-${{ matrix.ruby }}_${{ matrix.ffi || 'NATIVE' }} - path: logs +# Useful tool for troubleshooting, but the action introduces flakes as well, +# e.g. https://github.com/actions/upload-artifact/issues/569 +# - name: Archive log artifacts +# uses: actions/upload-artifact@v4 +# with: +# name: test-logs-${{ matrix.ruby }}_${{ matrix.ffi || 'NATIVE' }} +# path: logs linux-32bit: name: Linux 32-bit diff --git a/Cargo.bazel.lock b/Cargo.bazel.lock index 3855576c78..0d1aa040f8 100644 --- a/Cargo.bazel.lock +++ b/Cargo.bazel.lock @@ -1,9 +1,10 @@ { - "checksum": "8863e5b8f3da7cf4502f68bea0d455dec4834bf25ff070caaa58a8e1c5ea1a3d", + "checksum": "b4edbf28ea96b685a90948d9efaa14d55f0f01e27b5d774b3ecd6eff3c231517", "crates": { "aho-corasick 1.1.2": { "name": "aho-corasick", "version": "1.1.2", + "package_url": "https://github.com/BurntSushi/aho-corasick", "repository": { "Http": { "url": "https://static.crates.io/crates/aho-corasick/1.1.2/download", @@ -15,9 +16,12 @@ "Library": { "crate_name": "aho_corasick", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -46,11 +50,17 @@ "edition": "2021", "version": "1.1.2" }, - "license": "Unlicense OR MIT" + "license": "Unlicense OR MIT", + "license_ids": [ + "MIT", + "Unlicense" + ], + "license_file": "LICENSE-MIT" }, "autocfg 1.1.0": { "name": "autocfg", "version": "1.1.0", + "package_url": "https://github.com/cuviper/autocfg", "repository": { "Http": { "url": "https://static.crates.io/crates/autocfg/1.1.0/download", @@ -62,9 +72,12 @@ "Library": { "crate_name": "autocfg", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -76,20 +89,29 @@ "edition": "2015", "version": "1.1.0" }, - "license": "Apache-2.0 OR MIT" + "license": "Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "direct-cargo-bazel-deps 0.0.1": { "name": "direct-cargo-bazel-deps", "version": "0.0.1", + "package_url": null, "repository": null, "targets": [ { "Library": { "crate_name": "direct_cargo_bazel_deps", "crate_root": ".direct_cargo_bazel_deps.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -119,11 +141,14 @@ }, "version": "0.0.1" }, - "license": null + "license": null, + "license_ids": [], + "license_file": null }, "googletest 0.11.0": { "name": "googletest", "version": "0.11.0", + "package_url": "https://github.com/google/googletest-rust", "repository": { "Git": { "remote": "https://github.com/google/googletest-rust", @@ -138,9 +163,12 @@ "Library": { "crate_name": "googletest", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -178,11 +206,16 @@ }, "version": "0.11.0" }, - "license": "Apache-2.0" + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" }, "googletest_macro 0.11.0": { "name": "googletest_macro", "version": "0.11.0", + "package_url": "https://github.com/google/googletest-rust", "repository": { "Git": { "remote": "https://github.com/google/googletest-rust", @@ -197,9 +230,12 @@ "ProcMacro": { "crate_name": "googletest_macro", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -224,11 +260,16 @@ "edition": "2021", "version": "0.11.0" }, - "license": "Apache-2.0" + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "license_file": "LICENSE" }, "memchr 2.6.4": { "name": "memchr", "version": "2.6.4", + "package_url": "https://github.com/BurntSushi/memchr", "repository": { "Http": { "url": "https://static.crates.io/crates/memchr/2.6.4/download", @@ -240,9 +281,12 @@ "Library": { "crate_name": "memchr", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -262,11 +306,17 @@ "edition": "2021", "version": "2.6.4" }, - "license": "Unlicense OR MIT" + "license": "Unlicense OR MIT", + "license_ids": [ + "MIT", + "Unlicense" + ], + "license_file": "LICENSE-MIT" }, "num-traits 0.2.17": { "name": "num-traits", "version": "0.2.17", + "package_url": "https://github.com/rust-num/num-traits", "repository": { "Http": { "url": "https://static.crates.io/crates/num-traits/0.2.17/download", @@ -278,18 +328,24 @@ "Library": { "crate_name": "num_traits", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } }, { "BuildScript": { "crate_name": "build_script_build", "crate_root": "build.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -331,11 +387,17 @@ "selects": {} } }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "paste 1.0.14": { "name": "paste", "version": "1.0.14", + "package_url": "https://github.com/dtolnay/paste", "repository": { "Http": { "url": "https://static.crates.io/crates/paste/1.0.14/download", @@ -347,18 +409,24 @@ "ProcMacro": { "crate_name": "paste", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } }, { "BuildScript": { "crate_name": "build_script_build", "crate_root": "build.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -384,11 +452,17 @@ "**" ] }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "proc-macro2 1.0.69": { "name": "proc-macro2", "version": "1.0.69", + "package_url": "https://github.com/dtolnay/proc-macro2", "repository": { "Http": { "url": "https://static.crates.io/crates/proc-macro2/1.0.69/download", @@ -400,18 +474,24 @@ "Library": { "crate_name": "proc_macro2", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } }, { "BuildScript": { "crate_name": "build_script_build", "crate_root": "build.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -447,11 +527,17 @@ "**" ] }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "quote 1.0.33": { "name": "quote", "version": "1.0.33", + "package_url": "https://github.com/dtolnay/quote", "repository": { "Http": { "url": "https://static.crates.io/crates/quote/1.0.33/download", @@ -463,9 +549,12 @@ "Library": { "crate_name": "quote", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -493,11 +582,17 @@ "edition": "2018", "version": "1.0.33" }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "regex 1.10.0": { "name": "regex", "version": "1.10.0", + "package_url": "https://github.com/rust-lang/regex", "repository": { "Http": { "url": "https://static.crates.io/crates/regex/1.10.0/download", @@ -509,9 +604,12 @@ "Library": { "crate_name": "regex", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -566,11 +664,17 @@ "edition": "2021", "version": "1.10.0" }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "regex-automata 0.4.1": { "name": "regex-automata", "version": "0.4.1", + "package_url": "https://github.com/rust-lang/regex/tree/master/regex-automata", "repository": { "Http": { "url": "https://static.crates.io/crates/regex-automata/0.4.1/download", @@ -582,9 +686,12 @@ "Library": { "crate_name": "regex_automata", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -640,11 +747,17 @@ "edition": "2021", "version": "0.4.1" }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "regex-syntax 0.8.1": { "name": "regex-syntax", "version": "0.8.1", + "package_url": "https://github.com/rust-lang/regex/tree/master/regex-syntax", "repository": { "Http": { "url": "https://static.crates.io/crates/regex-syntax/0.8.1/download", @@ -656,9 +769,12 @@ "Library": { "crate_name": "regex_syntax", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -685,11 +801,17 @@ "edition": "2021", "version": "0.8.1" }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "rustversion 1.0.14": { "name": "rustversion", "version": "1.0.14", + "package_url": "https://github.com/dtolnay/rustversion", "repository": { "Http": { "url": "https://static.crates.io/crates/rustversion/1.0.14/download", @@ -701,18 +823,24 @@ "ProcMacro": { "crate_name": "rustversion", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } }, { "BuildScript": { "crate_name": "build_script_build", "crate_root": "build/build.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -738,11 +866,17 @@ "**" ] }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "syn 2.0.43": { "name": "syn", "version": "2.0.43", + "package_url": "https://github.com/dtolnay/syn", "repository": { "Http": { "url": "https://static.crates.io/crates/syn/2.0.43/download", @@ -754,9 +888,12 @@ "Library": { "crate_name": "syn", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -798,11 +935,17 @@ "edition": "2021", "version": "2.0.43" }, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" }, "unicode-ident 1.0.12": { "name": "unicode-ident", "version": "1.0.12", + "package_url": "https://github.com/dtolnay/unicode-ident", "repository": { "Http": { "url": "https://static.crates.io/crates/unicode-ident/1.0.12/download", @@ -814,9 +957,12 @@ "Library": { "crate_name": "unicode_ident", "crate_root": "src/lib.rs", - "srcs": [ - "**/*.rs" - ] + "srcs": { + "allow_empty": false, + "include": [ + "**/*.rs" + ] + } } } ], @@ -828,7 +974,13 @@ "edition": "2018", "version": "1.0.12" }, - "license": "(MIT OR Apache-2.0) AND Unicode-DFS-2016" + "license": "(MIT OR Apache-2.0) AND Unicode-DFS-2016", + "license_ids": [ + "Apache-2.0", + "MIT", + "Unicode-DFS-2016" + ], + "license_file": "LICENSE-APACHE" } }, "binary_crates": [], @@ -857,6 +1009,12 @@ "aarch64-unknown-linux-gnu": [ "aarch64-unknown-linux-gnu" ], + "aarch64-unknown-nixos-gnu": [ + "aarch64-unknown-nixos-gnu" + ], + "aarch64-unknown-nto-qnx710": [ + "aarch64-unknown-nto-qnx710" + ], "arm-unknown-linux-gnueabi": [ "arm-unknown-linux-gnueabi" ], @@ -926,8 +1084,16 @@ "x86_64-unknown-linux-gnu": [ "x86_64-unknown-linux-gnu" ], + "x86_64-unknown-nixos-gnu": [ + "x86_64-unknown-nixos-gnu" + ], "x86_64-unknown-none": [ "x86_64-unknown-none" ] - } -} + }, + "direct_deps": [ + "googletest 0.11.0", + "paste 1.0.14" + ], + "direct_dev_deps": [] +} \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 46207904a5..a59a2de4fb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,7 +23,7 @@ bazel_dep(name = "rules_pkg", version = "0.7.0") bazel_dep(name = "rules_python", version = "0.28.0") bazel_dep(name = "rules_rust", version = "0.45.1") bazel_dep(name = "platforms", version = "0.0.8") -bazel_dep(name = "zlib", version = "1.2.11") +bazel_dep(name = "zlib", version = "1.3.1") # TODO: remove after toolchain types are moved to protobuf bazel_dep(name = "rules_proto", version = "4.0.0") diff --git a/WORKSPACE b/WORKSPACE index 7bc8edb294..d84cb7f4b1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -202,8 +202,8 @@ fuzzing_py_deps_install_deps() http_archive( name = "rules_rust", - sha256 = "9ecd0f2144f0a24e6bc71ebcc50a1ee5128cedeceb32187004532c9710cb2334", - urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.29.1/rules_rust-v0.29.1.tar.gz"], + integrity = "sha256-F8U7+AC5MvMtPKGdLLnorVM84cDXKfDRgwd7/dq3rUY=", + urls = ["https://github.com/bazelbuild/rules_rust/releases/download/0.46.0/rules_rust-v0.46.0.tar.gz"], ) load("@rules_rust//rust:repositories.bzl", "rules_rust_dependencies", "rust_register_toolchains") diff --git a/compatibility/BUILD.bazel b/compatibility/BUILD.bazel index e62cb5b133..d55f61956e 100644 --- a/compatibility/BUILD.bazel +++ b/compatibility/BUILD.bazel @@ -16,5 +16,4 @@ java_runtime_conformance( java_runtime_conformance( name = "java_conformance_v3.25.0", gencode_version = "3.25.0", - tags = ["manual"], ) diff --git a/conformance/conformance.proto b/conformance/conformance.proto index d2a8c67016..2357d59ad6 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -86,8 +86,8 @@ message ConformanceRequest { // The full name for the test message to use; for the moment, either: // protobuf_test_messages.proto3.TestAllTypesProto3 or - // protobuf_test_messages.google.protobuf.TestAllTypesProto2 or - // protobuf_test_messages.editions.google.protobuf.TestAllTypesProto2 or + // protobuf_test_messages.proto2.TestAllTypesProto2 or + // protobuf_test_messages.editions.proto2.TestAllTypesProto2 or // protobuf_test_messages.editions.proto3.TestAllTypesProto3 or // protobuf_test_messages.editions.TestAllTypesEdition2023. string message_type = 4; diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs index 5f26120acf..f8a073b938 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.pb.cs @@ -495,8 +495,8 @@ namespace Conformance { /// /// The full name for the test message to use; for the moment, either: /// protobuf_test_messages.proto3.TestAllTypesProto3 or - /// protobuf_test_messages.google.protobuf.TestAllTypesProto2 or - /// protobuf_test_messages.editions.google.protobuf.TestAllTypesProto2 or + /// protobuf_test_messages.proto2.TestAllTypesProto2 or + /// protobuf_test_messages.editions.proto2.TestAllTypesProto2 or /// protobuf_test_messages.editions.proto3.TestAllTypesProto3 or /// protobuf_test_messages.editions.TestAllTypesEdition2023. /// diff --git a/csharp/src/Google.Protobuf.Test/testprotos.pb b/csharp/src/Google.Protobuf.Test/testprotos.pb index fa71270040..97f279d92e 100644 Binary files a/csharp/src/Google.Protobuf.Test/testprotos.pb and b/csharp/src/Google.Protobuf.Test/testprotos.pb differ diff --git a/csharp/src/Google.Protobuf/Reflection/Descriptor.pb.cs b/csharp/src/Google.Protobuf/Reflection/Descriptor.pb.cs index 38afae477f..ba3fee586c 100644 --- a/csharp/src/Google.Protobuf/Reflection/Descriptor.pb.cs +++ b/csharp/src/Google.Protobuf/Reflection/Descriptor.pb.cs @@ -8027,12 +8027,13 @@ namespace Google.Protobuf.Reflection { private global::Google.Protobuf.Reflection.FieldOptions.Types.CType ctype_; /// + /// NOTE: ctype is deprecated. Use `features.(pb.cpp).string_type` instead. /// The ctype option instructs the C++ code generator to use a different /// representation of the field than it normally would. See the specific /// options below. This option is only implemented to support use of /// [ctype=CORD] and [ctype=STRING] (the default) on non-repeated fields of - /// type "bytes" in the open source release -- sorry, we'll try to include - /// other types in a future version! + /// type "bytes" in the open source release. + /// TODO: make ctype actually deprecated. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] diff --git a/docs/options.md b/docs/options.md index a4986e0283..3f5ac61305 100644 --- a/docs/options.md +++ b/docs/options.md @@ -501,3 +501,8 @@ with info about your project (name and website) so we can add an entry for you. * Website: https://github.com/solo-io/protoc-gen-openapi * Extensions: 1188-1189 + +1. Wire enumMode + + * Website: https://square.github.io/wire/ + * Extensions: 1190 diff --git a/editions/BUILD b/editions/BUILD index 09838afc80..fd5ddddaf0 100644 --- a/editions/BUILD +++ b/editions/BUILD @@ -87,18 +87,6 @@ cc_library( ], ) -cc_proto_library( - name = "cpp_features_cc_proto", - testonly = True, - deps = ["//src/google/protobuf:cpp_features_proto"], -) - -cc_proto_library( - name = "java_features_cc_proto", - testonly = True, - deps = ["//java/core:java_features_proto"], -) - cc_test( name = "defaults_test", srcs = ["defaults_test.cc"], @@ -109,11 +97,11 @@ cc_test( ":test_defaults_future", ], deps = [ - ":cpp_features_cc_proto", ":defaults_test_embedded", - ":java_features_cc_proto", "//:protobuf", + "//java/core:java_features_cc_proto", "//src/google/protobuf", + "//src/google/protobuf:cpp_features_cc_proto", "//src/google/protobuf:port", "//src/google/protobuf:protobuf_lite", "//src/google/protobuf:test_textproto", diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index 7ca8a85636..d695ab0ad0 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -196,6 +196,12 @@ proto_library( deps = ["//:descriptor_proto"], ) +cc_proto_library( + name = "java_features_cc_proto", + visibility = ["//editions:__pkg__"], + deps = [":java_features_proto"], +) + filegroup( name = "java_features_proto_srcs", srcs = ["src/main/resources/google/protobuf/java_features.proto"], diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 08f5234af3..569fa26e0e 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -1901,7 +1901,9 @@ public final class Descriptors { } } - if (getJavaType() == JavaType.MESSAGE) { + // Use raw type since inferred type considers messageType which may not be fully cross + // linked yet. + if (type.getJavaType() == JavaType.MESSAGE) { if (!(typeDescriptor instanceof Descriptor)) { throw new DescriptorValidationException( this, '\"' + proto.getTypeName() + "\" is not a message type."); @@ -1911,7 +1913,7 @@ public final class Descriptors { if (proto.hasDefaultValue()) { throw new DescriptorValidationException(this, "Messages can't have default values."); } - } else if (getJavaType() == JavaType.ENUM) { + } else if (type.getJavaType() == JavaType.ENUM) { if (!(typeDescriptor instanceof EnumDescriptor)) { throw new DescriptorValidationException( this, '\"' + proto.getTypeName() + "\" is not an enum type."); @@ -1921,7 +1923,7 @@ public final class Descriptors { throw new DescriptorValidationException(this, "Field with primitive type has type_name."); } } else { - if (getJavaType() == JavaType.MESSAGE || getJavaType() == JavaType.ENUM) { + if (type.getJavaType() == JavaType.MESSAGE || type.getJavaType() == JavaType.ENUM) { throw new DescriptorValidationException( this, "Field with message or enum type missing type_name."); } @@ -1942,7 +1944,7 @@ public final class Descriptors { } try { - switch (getType()) { + switch (type) { case INT32: case SINT32: case SFIXED32: @@ -2017,7 +2019,7 @@ public final class Descriptors { if (isRepeated()) { defaultValue = Collections.emptyList(); } else { - switch (getJavaType()) { + switch (type.getJavaType()) { case ENUM: // We guarantee elsewhere that an enum type always has at least // one possible value. @@ -2027,7 +2029,7 @@ public final class Descriptors { defaultValue = null; break; default: - defaultValue = getJavaType().defaultDefault; + defaultValue = type.getJavaType().defaultDefault; break; } } @@ -2783,10 +2785,30 @@ public final class Descriptors { public abstract FileDescriptor getFile(); void resolveFeatures(FeatureSet unresolvedFeatures) throws DescriptorValidationException { - // Unknown java features may be passed by users via public buildFrom but should not occur from - // generated code. - if (!unresolvedFeatures.getUnknownFields().isEmpty() - && unresolvedFeatures.getUnknownFields().hasField(JavaFeaturesProto.java_.getNumber())) { + if (this.parent != null + && unresolvedFeatures.equals(FeatureSet.getDefaultInstance()) + && !hasInferredLegacyProtoFeatures()) { + this.features = this.parent.features; + validateFeatures(); + return; + } + + // Java features from a custom pool (i.e. buildFrom) may end up in unknown fields or + // use a different descriptor from the generated pool used by the Java runtime. + boolean hasPossibleCustomJavaFeature = false; + for (FieldDescriptor f : unresolvedFeatures.getExtensionFields().keySet()) { + if (f.getNumber() == JavaFeaturesProto.java_.getNumber() + && f != JavaFeaturesProto.java_.getDescriptor()) { + hasPossibleCustomJavaFeature = true; + continue; + } + } + boolean hasPossibleUnknownJavaFeature = + !unresolvedFeatures.getUnknownFields().isEmpty() + && unresolvedFeatures + .getUnknownFields() + .hasField(JavaFeaturesProto.java_.getNumber()); + if (hasPossibleCustomJavaFeature || hasPossibleUnknownJavaFeature) { ExtensionRegistry registry = ExtensionRegistry.newInstance(); registry.add(JavaFeaturesProto.java_); ByteString bytes = unresolvedFeatures.toByteString(); @@ -2797,14 +2819,7 @@ public final class Descriptors { this, "Failed to parse features with Java feature extension registry.", e); } } - - if (this.parent != null - && unresolvedFeatures.equals(FeatureSet.getDefaultInstance()) - && !hasInferredLegacyProtoFeatures()) { - this.features = this.parent.features; - validateFeatures(); - return; - } + FeatureSet.Builder features; if (this.parent == null) { Edition edition = getFile().getEdition(); diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java index 9576051d00..aeeeb94e3d 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessage.java @@ -1025,10 +1025,28 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial /** * Used by subclasses to serialize extensions. Extension ranges may be interleaved with field - * numbers, but we must write them in canonical (sorted by field number) order. ExtensionWriter - * helps us write individual ranges of extensions at once. + * numbers, but we must write them in canonical (sorted by field number) order. + * ExtensionSerializer helps us write individual ranges of extensions at once. */ - protected class ExtensionWriter { + protected interface ExtensionSerializer { + public void writeUntil(final int end, final CodedOutputStream output) throws IOException; + } + + /** No-op implementation that writes nothing, for messages with no extensions. */ + private static final class NoOpExtensionSerializer implements ExtensionSerializer { + // Singleton instance so we can avoid allocating a new one for each message serialization. + private static final NoOpExtensionSerializer INSTANCE = new NoOpExtensionSerializer(); + + @Override + public void writeUntil(final int end, final CodedOutputStream output) { + // no-op + } + } + + /** + * ExtensionSerializer that writes extensions from the FieldSet, for messages with extensions. + */ + protected class ExtensionWriter implements ExtensionSerializer { // Imagine how much simpler this code would be if Java iterators had // a way to get the next element without advancing the iterator. @@ -1043,6 +1061,7 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial this.messageSetWireFormat = messageSetWireFormat; } + @Override public void writeUntil(final int end, final CodedOutputStream output) throws IOException { while (next != null && next.getKey().getNumber() < end) { FieldDescriptor descriptor = next.getKey(); @@ -1075,14 +1094,39 @@ public abstract class GeneratedMessage extends AbstractMessage implements Serial } } + /** + * For compatibility with older gencode. + * + *

TODO Remove this in the next breaking release. + * + * @deprecated Use {@link newExtensionSerializer()} instead. + */ + @Deprecated protected ExtensionWriter newExtensionWriter() { return new ExtensionWriter(false); } + protected ExtensionSerializer newExtensionSerializer() { + // Avoid allocation in the common case of no extensions. + if (extensions.isEmpty()) { + return NoOpExtensionSerializer.INSTANCE; + } + return new ExtensionWriter(false); + } + + // TODO: Remove, replace with newMessageSetExtensionSerializer(). protected ExtensionWriter newMessageSetExtensionWriter() { return new ExtensionWriter(true); } + protected ExtensionSerializer newMessageSetExtensionSerializer() { + // Avoid allocation in the common case of no extensions. + if (extensions.isEmpty()) { + return NoOpExtensionSerializer.INSTANCE; + } + return new ExtensionWriter(true); + } + /** Called by subclasses to compute the size of extensions. */ protected int extensionsSerializedSize() { return extensions.getSerializedSize(); diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java new file mode 100644 index 0000000000..9071495866 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -0,0 +1,83 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +package com.google.protobuf; + +import java.util.List; + +/** + * Stub for GeneratedMessageV3 wrapping GeneratedMessage for compatibility with older gencode. + * + * @deprecated This class is deprecated, and slated for removal in the next Java breaking change + * (5.x in 2025 Q1). Users should update gencode to >= 4.26.x which uses GeneratedMessage + * instead of GeneratedMessageV3. + */ +@Deprecated +public abstract class GeneratedMessageV3 extends GeneratedMessage { + private static final long serialVersionUID = 1L; + + protected GeneratedMessageV3() { + super(); + } + + protected GeneratedMessageV3(Builder builder) { + super(builder); + } + + /** + * Stub for GeneratedMessageV3.ExtendableBuilder wrapping GeneratedMessage.ExtendableBuilder for + * compatibility with older gencode. + * + * @deprecated This class is deprecated, and slated for removal in the next Java breaking change + * (5.x in 2025 Q1). Users should update gencode to >= 4.26.x which uses + * GeneratedMessage.ExtendableBuilder instead of GeneratedMessageV3.ExtendableBuilder. + */ + @Deprecated + public abstract static class ExtendableBuilder< + MessageT extends ExtendableMessage, + BuilderT extends ExtendableBuilder> + extends GeneratedMessage.ExtendableBuilder + implements GeneratedMessage.ExtendableMessageOrBuilder { + protected ExtendableBuilder() { + super(); + } + + protected ExtendableBuilder(BuilderParent parent) { + super(parent); + } + + // Support old gencode override method removed in + // https://github.com/protocolbuffers/protobuf/commit/7bff169d32710b143951ec6ce2c4ea9a56e2ad24 + public BuilderT setExtension( + final GeneratedMessage.GeneratedExtension extension, final T value) { + return setExtension((ExtensionLite) extension, value); + } + + // Support old gencode override method removed in + // https://github.com/protocolbuffers/protobuf/commit/7bff169d32710b143951ec6ce2c4ea9a56e2ad24 + public BuilderT setExtension( + final GeneratedMessage.GeneratedExtension> extension, + final int index, + final T value) { + return setExtension((ExtensionLite>) extension, index, value); + } + + // Support old gencode override method removed in + // https://github.com/protocolbuffers/protobuf/commit/7bff169d32710b143951ec6ce2c4ea9a56e2ad24 + public BuilderT addExtension( + final GeneratedMessage.GeneratedExtension> extension, final T value) { + return addExtension((ExtensionLite>) extension, value); + } + + // Support old gencode override method removed in + // https://github.com/protocolbuffers/protobuf/commit/7bff169d32710b143951ec6ce2c4ea9a56e2ad24 + public BuilderT clearExtension( + final GeneratedMessage.GeneratedExtension extension) { + return clearExtension((ExtensionLite) extension); + } + } +} diff --git a/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java b/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java new file mode 100644 index 0000000000..14a47a5d46 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java @@ -0,0 +1,34 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +package com.google.protobuf; + +import java.util.List; + +/** + * Stub for RepeatedFieldBuilderV3 wrapping RepeatedFieldBuilder for compatibility with older + * gencode. + * + * @deprecated This class is deprecated, and slated for removal in the next breaking change. Users + * should update gencode to >= 4.26.x which replaces RepeatedFieldBuilderV3 with + * RepeatedFieldBuilder. + */ +@Deprecated +public class RepeatedFieldBuilderV3< + MType extends GeneratedMessage, + BType extends GeneratedMessage.Builder, + IType extends MessageOrBuilder> + extends RepeatedFieldBuilder { + + public RepeatedFieldBuilderV3( + List messages, + boolean isMessagesListMutable, + GeneratedMessage.BuilderParent parent, + boolean isClean) { + super(messages, isMessagesListMutable, parent, isClean); + } +} diff --git a/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java b/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java new file mode 100644 index 0000000000..3c6b151de2 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java @@ -0,0 +1,28 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2024 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +package com.google.protobuf; + +/** + * Stub for SingleFieldBuilderV3 wrapping SingleFieldBuilder for compatibility with older gencode. + * + * @deprecated This class is deprecated, and slated for removal in the next breaking change. Users + * should update gencode to >= 4.26.x which replaces SingleFieldBuilderV3 with + * SingleFieldBuilder. + */ +@Deprecated +public class SingleFieldBuilderV3< + MType extends GeneratedMessage, + BType extends GeneratedMessage.Builder, + IType extends MessageOrBuilder> + extends SingleFieldBuilder { + + public SingleFieldBuilderV3( + MType message, GeneratedMessage.BuilderParent parent, boolean isClean) { + super(message, parent, isClean); + } +} diff --git a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java index 508344d3bf..5eca109d77 100644 --- a/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/core/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -352,8 +352,9 @@ public class DescriptorsTest { } @Test - public void testProto2FieldDescriptorLegacyEnumFieldTreatedAsClosed() throws Exception { - // Make an open enum definition. + public void testFieldDescriptorLegacyEnumFieldTreatedAsOpen() throws Exception { + // Make an open enum definition and message that treats enum fields as open. + FileDescriptorProto openEnumFile = FileDescriptorProto.newBuilder() .setName("open_enum.proto") @@ -367,30 +368,9 @@ public class DescriptorsTest { .setNumber(0) .build()) .build()) - .build(); - FileDescriptor openFileDescriptor = - Descriptors.FileDescriptor.buildFrom(openEnumFile, new FileDescriptor[0]); - EnumDescriptor openEnum = openFileDescriptor.getEnumTypes().get(0); - assertThat(openEnum.isClosed()).isFalse(); - - // Create a message that treats enum fields as closed. - FileDescriptorProto closedEnumFile = - FileDescriptorProto.newBuilder() - .setName("closed_enum_field.proto") - .addDependency("open_enum.proto") - .setSyntax("proto2") - .addEnumType( - EnumDescriptorProto.newBuilder() - .setName("TestEnum") - .addValue( - EnumValueDescriptorProto.newBuilder() - .setName("TestEnum_VALUE0") - .setNumber(0) - .build()) - .build()) .addMessageType( DescriptorProto.newBuilder() - .setName("TestClosedEnumField") + .setName("TestOpenEnumField") .addField( FieldDescriptorProto.newBuilder() .setName("int_field") @@ -406,32 +386,21 @@ public class DescriptorsTest { .setTypeName("TestEnumOpen") .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) .build()) - .addField( - FieldDescriptorProto.newBuilder() - .setName("closed_enum") - .setNumber(3) - .setType(FieldDescriptorProto.Type.TYPE_ENUM) - .setTypeName("TestEnum") - .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) - .build()) .build()) .build(); - Descriptor closedMessage = - Descriptors.FileDescriptor.buildFrom( - closedEnumFile, new FileDescriptor[] {openFileDescriptor}) - .getMessageTypes() - .get(0); - assertThat(closedMessage.findFieldByName("int_field").legacyEnumFieldTreatedAsClosed()) + FileDescriptor openEnumFileDescriptor = + Descriptors.FileDescriptor.buildFrom(openEnumFile, new FileDescriptor[0]); + Descriptor openMessage = openEnumFileDescriptor.getMessageTypes().get(0); + EnumDescriptor openEnum = openEnumFileDescriptor.findEnumTypeByName("TestEnumOpen"); + assertThat(openEnum.isClosed()).isFalse(); + assertThat(openMessage.findFieldByName("int_field").legacyEnumFieldTreatedAsClosed()) + .isFalse(); + assertThat(openMessage.findFieldByName("open_enum").legacyEnumFieldTreatedAsClosed()) .isFalse(); - - assertThat(closedMessage.findFieldByName("closed_enum").legacyEnumFieldTreatedAsClosed()) - .isTrue(); - assertThat(closedMessage.findFieldByName("open_enum").legacyEnumFieldTreatedAsClosed()) - .isTrue(); } @Test - public void testEditionFieldDescriptorLegacyEnumFieldTreatedAsClosed() throws Exception { + public void testEditionFieldDescriptorLegacyEnumFieldTreatedAsClosedUnknown() throws Exception { // Make an open enum definition. FileDescriptorProto openEnumFile = FileDescriptorProto.newBuilder() @@ -536,12 +505,19 @@ public class DescriptorsTest { } @Test - public void testFieldDescriptorLegacyEnumFieldTreatedAsOpen() throws Exception { - // Make an open enum definition and message that treats enum fields as open. + public void testEditionFieldDescriptorLegacyEnumFieldTreatedAsClosedCustomPool() + throws Exception { + + FileDescriptor javaFeaturesDescriptor = + Descriptors.FileDescriptor.buildFrom( + JavaFeaturesProto.getDescriptor().toProto(), + new FileDescriptor[] {DescriptorProtos.getDescriptor()}); + // Make an open enum definition. FileDescriptorProto openEnumFile = FileDescriptorProto.newBuilder() .setName("open_enum.proto") - .setSyntax("proto3") + .setSyntax("editions") + .setEdition(Edition.EDITION_2023) .addEnumType( EnumDescriptorProto.newBuilder() .setName("TestEnumOpen") @@ -551,9 +527,38 @@ public class DescriptorsTest { .setNumber(0) .build()) .build()) + .build(); + FileDescriptor openFileDescriptor = + Descriptors.FileDescriptor.buildFrom(openEnumFile, new FileDescriptor[0]); + EnumDescriptor openEnum = openFileDescriptor.getEnumTypes().get(0); + assertThat(openEnum.isClosed()).isFalse(); + + // Create a message that treats enum fields as closed. + FileDescriptorProto editionsClosedEnumFile = + FileDescriptorProto.newBuilder() + .setName("editions_closed_enum_field.proto") + .addDependency("open_enum.proto") + .setSyntax("editions") + .setEdition(Edition.EDITION_2023) + .setOptions( + FileOptions.newBuilder() + .setFeatures( + DescriptorProtos.FeatureSet.newBuilder() + .setEnumType(DescriptorProtos.FeatureSet.EnumType.CLOSED) + .build()) + .build()) + .addEnumType( + EnumDescriptorProto.newBuilder() + .setName("TestEnum") + .addValue( + EnumValueDescriptorProto.newBuilder() + .setName("TestEnum_VALUE0") + .setNumber(0) + .build()) + .build()) .addMessageType( DescriptorProto.newBuilder() - .setName("TestOpenEnumField") + .setName("TestClosedEnumField") .addField( FieldDescriptorProto.newBuilder() .setName("int_field") @@ -568,18 +573,53 @@ public class DescriptorsTest { .setType(FieldDescriptorProto.Type.TYPE_ENUM) .setTypeName("TestEnumOpen") .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) + .setOptions( + DescriptorProtos.FieldOptions.newBuilder() + .setFeatures( + DescriptorProtos.FeatureSet.newBuilder() + .setExtension( + // Extension cannot be directly set using custom + // descriptor, so set using generated for now. + JavaFeaturesProto.java_, + JavaFeaturesProto.JavaFeatures.newBuilder() + .setLegacyClosedEnum(true) + .build()) + .build()) + .build()) + .build()) + .addField( + FieldDescriptorProto.newBuilder() + .setName("closed_enum") + .setNumber(3) + .setType(FieldDescriptorProto.Type.TYPE_ENUM) + .setTypeName("TestEnum") + .setLabel(FieldDescriptorProto.Label.LABEL_OPTIONAL) .build()) .build()) .build(); - FileDescriptor openEnumFileDescriptor = - Descriptors.FileDescriptor.buildFrom(openEnumFile, new FileDescriptor[0]); - Descriptor openMessage = openEnumFileDescriptor.getMessageTypes().get(0); - EnumDescriptor openEnum = openEnumFileDescriptor.findEnumTypeByName("TestEnumOpen"); - assertThat(openEnum.isClosed()).isFalse(); - assertThat(openMessage.findFieldByName("int_field").legacyEnumFieldTreatedAsClosed()) - .isFalse(); - assertThat(openMessage.findFieldByName("open_enum").legacyEnumFieldTreatedAsClosed()) + // Reparse using custom java features descriptor. + ExtensionRegistry registry = ExtensionRegistry.newInstance(); + registry.add( + javaFeaturesDescriptor.getExtensions().get(0), + DynamicMessage.getDefaultInstance( + javaFeaturesDescriptor.getExtensions().get(0).getMessageType())); + editionsClosedEnumFile = + FileDescriptorProto.parseFrom(editionsClosedEnumFile.toByteString(), registry); + Descriptor editionsClosedMessage = + Descriptors.FileDescriptor.buildFrom( + editionsClosedEnumFile, + new FileDescriptor[] {openFileDescriptor, javaFeaturesDescriptor}) + .getMessageTypes() + .get(0); + assertThat( + editionsClosedMessage.findFieldByName("int_field").legacyEnumFieldTreatedAsClosed()) .isFalse(); + assertThat( + editionsClosedMessage.findFieldByName("closed_enum").legacyEnumFieldTreatedAsClosed()) + .isTrue(); + assertThat( + editionsClosedMessage.findFieldByName("open_enum").legacyEnumFieldTreatedAsClosed()) + .isTrue(); } @Test diff --git a/php/ext/google/protobuf/php-upb.c b/php/ext/google/protobuf/php-upb.c index 022fab0104..8a24966067 100644 --- a/php/ext/google/protobuf/php-upb.c +++ b/php/ext/google/protobuf/php-upb.c @@ -487,7 +487,7 @@ void upb_Status_VAppendErrorFormat(upb_Status* status, const char* fmt, // Must be last. -extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty); +extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken); static const upb_MiniTableSubInternal google_protobuf_FileDescriptorSet_submsgs[1] = { {.UPB_PRIVATE(submsg) = &google__protobuf__FileDescriptorProto_msg_init_ptr}, }; @@ -12615,7 +12615,12 @@ char* upb_MtDataEncoder_EndEnum(upb_MtDataEncoder* e, char* ptr) { // Must be last. -// A MiniTable for an empty message, used for unlinked sub-messages. +// A MiniTable for an empty message, used for unlinked sub-messages that are +// built via MiniDescriptors. Messages that use this MiniTable may possibly +// be linked later, in which case this MiniTable will be replaced with a real +// one. This pattern is known as "dynamic tree shaking", and it introduces +// complication because sub-messages may either be the "empty" type or the +// "real" type. A tagged bit indicates the difference. const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { .UPB_PRIVATE(subs) = NULL, .UPB_PRIVATE(fields) = NULL, @@ -12627,6 +12632,21 @@ const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { .UPB_PRIVATE(required_count) = 0, }; +// A MiniTable for a statically tree shaken message. Messages that use this +// MiniTable are guaranteed to remain unlinked; unlike the empty message, this +// MiniTable is never replaced, which greatly simplifies everything, because the +// type of a sub-message is always known, without consulting a tagged bit. +const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken) = { + .UPB_PRIVATE(subs) = NULL, + .UPB_PRIVATE(fields) = NULL, + .UPB_PRIVATE(size) = sizeof(struct upb_Message), + .UPB_PRIVATE(field_count) = 0, + .UPB_PRIVATE(ext) = kUpb_ExtMode_NonExtendable, + .UPB_PRIVATE(dense_below) = 0, + .UPB_PRIVATE(table_mask) = -1, + .UPB_PRIVATE(required_count) = 0, +}; + // Must be last. diff --git a/php/ext/google/protobuf/protobuf.c b/php/ext/google/protobuf/protobuf.c index 1f36ed0952..1f6889ff74 100644 --- a/php/ext/google/protobuf/protobuf.c +++ b/php/ext/google/protobuf/protobuf.c @@ -143,11 +143,11 @@ static PHP_GINIT_FUNCTION(protobuf) { protobuf_globals->global_symtab = NULL; } static PHP_RINIT_FUNCTION(protobuf) { // Create the global generated pool. // Reuse the symtab (if any) left to us by the last request. - upb_DefPool* symtab = PROTOBUF_G(global_symtab); - if (!symtab) { - PROTOBUF_G(global_symtab) = symtab = upb_DefPool_New(); - zend_hash_init(&PROTOBUF_G(name_msg_cache), 64, NULL, NULL, 0); - zend_hash_init(&PROTOBUF_G(name_enum_cache), 64, NULL, NULL, 0); + if (!PROTOBUF_G(global_symtab)) { + zend_bool persistent = PROTOBUF_G(keep_descriptor_pool_after_request); + PROTOBUF_G(global_symtab) = upb_DefPool_New(); + zend_hash_init(&PROTOBUF_G(name_msg_cache), 64, NULL, NULL, persistent); + zend_hash_init(&PROTOBUF_G(name_enum_cache), 64, NULL, NULL, persistent); } zend_hash_init(&PROTOBUF_G(object_cache), 64, NULL, NULL, 0); @@ -308,7 +308,7 @@ zend_module_entry protobuf_module_entry = { protobuf_functions, // function list PHP_MINIT(protobuf), // process startup PHP_MSHUTDOWN(protobuf), // process shutdown - PHP_RINIT(protobuf), // request shutdown + PHP_RINIT(protobuf), // request startup PHP_RSHUTDOWN(protobuf), // request shutdown NULL, // extension info PHP_PROTOBUF_VERSION, // extension version diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index a852e49a44..ae2aa5e728 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -54,11 +54,11 @@ def protobuf_deps(): http_archive( name = "zlib", build_file = Label("//:third_party/zlib.BUILD"), - sha256 = "d14c38e313afc35a9a8760dadf26042f51ea0f5d154b0630a31da0540107fb98", - strip_prefix = "zlib-1.2.13", + sha256 = "38ef96b8dfe510d42707d9c781877914792541133e1870841463bfa73f883e32", + strip_prefix = "zlib-1.3.1", urls = [ - "https://github.com/madler/zlib/releases/download/v1.2.13/zlib-1.2.13.tar.xz", - "https://zlib.net/zlib-1.2.13.tar.xz", + "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.xz", + "https://zlib.net/zlib-1.3.1.tar.xz", ], ) diff --git a/python/message.c b/python/message.c index 8e418f0d7e..719f8581d8 100644 --- a/python/message.c +++ b/python/message.c @@ -1350,7 +1350,9 @@ PyObject* PyUpb_Message_MergeFromString(PyObject* _self, PyObject* arg) { upb_Decode(buf, size, self->ptr.msg, layout, extreg, options, arena); Py_XDECREF(bytes); if (status != kUpb_DecodeStatus_Ok) { - PyErr_Format(state->decode_error_class, "Error parsing message"); + PyErr_Format(state->decode_error_class, + "Error parsing message with type '%s'", + upb_MessageDef_FullName(msgdef)); return NULL; } PyUpb_Message_SyncSubobjs(self); diff --git a/ruby/ext/google/protobuf_c/ruby-upb.c b/ruby/ext/google/protobuf_c/ruby-upb.c index 2ffd880477..9c007d789d 100644 --- a/ruby/ext/google/protobuf_c/ruby-upb.c +++ b/ruby/ext/google/protobuf_c/ruby-upb.c @@ -487,7 +487,7 @@ void upb_Status_VAppendErrorFormat(upb_Status* status, const char* fmt, // Must be last. -extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty); +extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken); static const upb_MiniTableSubInternal google_protobuf_FileDescriptorSet_submsgs[1] = { {.UPB_PRIVATE(submsg) = &google__protobuf__FileDescriptorProto_msg_init_ptr}, }; @@ -12103,7 +12103,12 @@ char* upb_MtDataEncoder_EndEnum(upb_MtDataEncoder* e, char* ptr) { // Must be last. -// A MiniTable for an empty message, used for unlinked sub-messages. +// A MiniTable for an empty message, used for unlinked sub-messages that are +// built via MiniDescriptors. Messages that use this MiniTable may possibly +// be linked later, in which case this MiniTable will be replaced with a real +// one. This pattern is known as "dynamic tree shaking", and it introduces +// complication because sub-messages may either be the "empty" type or the +// "real" type. A tagged bit indicates the difference. const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { .UPB_PRIVATE(subs) = NULL, .UPB_PRIVATE(fields) = NULL, @@ -12115,6 +12120,21 @@ const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { .UPB_PRIVATE(required_count) = 0, }; +// A MiniTable for a statically tree shaken message. Messages that use this +// MiniTable are guaranteed to remain unlinked; unlike the empty message, this +// MiniTable is never replaced, which greatly simplifies everything, because the +// type of a sub-message is always known, without consulting a tagged bit. +const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken) = { + .UPB_PRIVATE(subs) = NULL, + .UPB_PRIVATE(fields) = NULL, + .UPB_PRIVATE(size) = sizeof(struct upb_Message), + .UPB_PRIVATE(field_count) = 0, + .UPB_PRIVATE(ext) = kUpb_ExtMode_NonExtendable, + .UPB_PRIVATE(dense_below) = 0, + .UPB_PRIVATE(table_mask) = -1, + .UPB_PRIVATE(required_count) = 0, +}; + // Must be last. diff --git a/rust/cpp.rs b/rust/cpp.rs index 44451db1cc..de038679f6 100644 --- a/rust/cpp.rs +++ b/rust/cpp.rs @@ -9,8 +9,8 @@ use crate::__internal::{Enum, Private}; use crate::{ - Map, MapIter, Mut, ProtoStr, Proxied, ProxiedInMapValue, ProxiedInRepeated, Repeated, - RepeatedMut, RepeatedView, View, + Map, MapIter, Mut, ProtoBytes, ProtoStr, ProtoString, Proxied, ProxiedInMapValue, + ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, }; use core::fmt::Debug; use paste::paste; @@ -75,6 +75,25 @@ pub type RawRepeatedField = NonNull<_opaque_pointees::RawRepeatedFieldData>; /// A raw pointer to the underlying arena for this runtime. pub type RawMap = NonNull<_opaque_pointees::RawMapData>; +/// Kernel-specific owned `string` and `bytes` field type. +// TODO - b/334788521: Allocate this on the C++ side (maybe as a std::string), and move the +// std::string instead of copying the string_view (which we currently do). +#[derive(Debug)] +pub struct InnerProtoString(Box<[u8]>); + +impl InnerProtoString { + pub(crate) fn as_bytes(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl From<&[u8]> for InnerProtoString { + fn from(val: &[u8]) -> Self { + let owned_copy: Box<[u8]> = val.into(); + InnerProtoString(owned_copy) + } +} + /// Represents an ABI-stable version of `NonNull<[u8]>`/`string_view` (a /// borrowed slice of bytes) for FFI use only. /// @@ -164,6 +183,28 @@ impl SerializedData { fn as_mut_ptr(&mut self) -> *mut [u8] { ptr::slice_from_raw_parts_mut(self.data.as_ptr(), self.len) } + + /// Converts into a Vec. + pub fn into_vec(self) -> Vec { + // We need to prevent self from being dropped, because we are going to transfer + // ownership of self.data to the Vec. + let s = std::mem::ManuallyDrop::new(self); + + unsafe { + // SAFETY: + // - `data` was allocated by the Rust global allocator. + // - `data` was allocated with an alignment of 1 for u8. + // - The allocated size was `len`. + // - The length and capacity are equal. + // - All `len` bytes are initialized. + // - The capacity (`len` in this case) is the size the pointer was allocated + // with. + // - The allocated size is no more than isize::MAX, because the protobuf + // serializer will refuse to serialize a message if the output would exceed + // 2^31 - 1 bytes. + Vec::::from_raw_parts(s.data.as_ptr(), s.len, s.len) + } + } } impl Deref for SerializedData { @@ -175,13 +216,6 @@ impl Deref for SerializedData { } } -// TODO: remove after IntoProxied has been implemented for bytes. -impl AsRef<[u8]> for SerializedData { - fn as_ref(&self) -> &[u8] { - self - } -} - impl Drop for SerializedData { fn drop(&mut self) { // SAFETY: `data` was allocated by the Rust global allocator with a @@ -354,15 +388,15 @@ macro_rules! impl_cpp_type_conversions_for_scalars { impl_cpp_type_conversions_for_scalars!(i32, u32, i64, u64, f32, f64, bool); -impl CppTypeConversions for ProtoStr { +impl CppTypeConversions for ProtoString { type ElemType = PtrAndLen; - fn elem_to_view<'msg>(v: PtrAndLen) -> View<'msg, ProtoStr> { + fn elem_to_view<'msg>(v: PtrAndLen) -> View<'msg, ProtoString> { ptrlen_to_str(v) } } -impl CppTypeConversions for [u8] { +impl CppTypeConversions for ProtoBytes { type ElemType = PtrAndLen; fn elem_to_view<'msg>(v: Self::ElemType) -> View<'msg, Self> { @@ -370,10 +404,6 @@ impl CppTypeConversions for [u8] { } } -// This type alias is used so macros can generate valid extern "C" symbol names -// for functions working with [u8] types. -type Bytes = [u8]; - macro_rules! impl_repeated_primitives { (@impl $($t:ty => [ $new_thunk:ident, @@ -470,7 +500,7 @@ macro_rules! impl_repeated_primitives { }; } -impl_repeated_primitives!(i32, u32, i64, u64, f32, f64, bool, ProtoStr, Bytes); +impl_repeated_primitives!(i32, u32, i64, u64, f32, f64, bool, ProtoString, ProtoBytes); /// Cast a `RepeatedView` to `RepeatedView`. pub fn cast_enum_repeated_view( @@ -764,8 +794,8 @@ macro_rules! impl_ProxiedInMapValue_for_key_types { i64, i64, identity, identity; u64, u64, identity, identity; bool, bool, identity, identity; - ProtoStr, PtrAndLen, str_to_ptrlen, ptrlen_to_str; - Bytes, PtrAndLen, bytes_to_ptrlen, ptrlen_to_bytes; + ProtoString, PtrAndLen, str_to_ptrlen, ptrlen_to_str; + ProtoBytes, PtrAndLen, bytes_to_ptrlen, ptrlen_to_bytes; ); )* } @@ -778,7 +808,7 @@ impl_ProxiedInMapValue_for_key_types!( i64, i64, identity, identity; u64, u64, identity, identity; bool, bool, identity, identity; - ProtoStr, PtrAndLen, str_to_ptrlen, ptrlen_to_str; + ProtoString, PtrAndLen, str_to_ptrlen, ptrlen_to_str; ); #[cfg(test)] diff --git a/rust/cpp_kernel/cpp_api.cc b/rust/cpp_kernel/cpp_api.cc index a4ba3fe37f..d41a485075 100644 --- a/rust/cpp_kernel/cpp_api.cc +++ b/rust/cpp_kernel/cpp_api.cc @@ -100,8 +100,8 @@ expose_repeated_field_methods(int64_t, i64); r->Reserve(r->size() + additional); \ } -expose_repeated_ptr_field_methods(ProtoStr); -expose_repeated_ptr_field_methods(Bytes); +expose_repeated_ptr_field_methods(ProtoString); +expose_repeated_ptr_field_methods(ProtoBytes); #undef expose_repeated_field_methods #undef expose_repeated_ptr_field_methods @@ -126,11 +126,11 @@ __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE(uint64_t, u64, uint64_t, __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE(int64_t, i64, int64_t, value, cpp_value); __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE( - std::string, Bytes, google::protobuf::rust_internal::PtrAndLen, + std::string, ProtoBytes, google::protobuf::rust_internal::PtrAndLen, std::string(value.ptr, value.len), google::protobuf::rust_internal::PtrAndLen(cpp_value.data(), cpp_value.size())); __PB_RUST_EXPOSE_SCALAR_MAP_METHODS_FOR_VALUE_TYPE( - std::string, ProtoStr, google::protobuf::rust_internal::PtrAndLen, + std::string, ProtoString, google::protobuf::rust_internal::PtrAndLen, std::string(value.ptr, value.len), google::protobuf::rust_internal::PtrAndLen(cpp_value.data(), cpp_value.size())); diff --git a/rust/cpp_kernel/cpp_api.h b/rust/cpp_kernel/cpp_api.h index 9345a01c3f..24765f63a7 100644 --- a/rust/cpp_kernel/cpp_api.h +++ b/rust/cpp_kernel/cpp_api.h @@ -156,7 +156,7 @@ struct PtrAndLen { value_ty, rust_value_ty, ffi_value_ty, \ to_cpp_value, to_ffi_value); \ __PB_RUST_EXPOSE_SCALAR_MAP_METHODS( \ - std::string, ProtoStr, google::protobuf::rust_internal::PtrAndLen, \ + std::string, ProtoString, google::protobuf::rust_internal::PtrAndLen, \ std::string(key.ptr, key.len), \ google::protobuf::rust_internal::PtrAndLen(cpp_key.data(), cpp_key.size()), \ value_ty, rust_value_ty, ffi_value_ty, to_cpp_value, to_ffi_value); diff --git a/rust/map.rs b/rust/map.rs index 3ceb2a7aac..dc852408c2 100644 --- a/rust/map.rs +++ b/rust/map.rs @@ -458,7 +458,7 @@ where #[cfg(test)] mod tests { use super::*; - use crate::ProtoStr; + use crate::{ProtoBytes, ProtoStr, ProtoString}; use googletest::prelude::*; #[test] @@ -489,7 +489,7 @@ mod tests { #[test] fn test_proxied_str() { - let mut map: Map = Map::new(); + let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); map_mut.insert("a", "b"); @@ -514,7 +514,7 @@ mod tests { #[test] fn test_proxied_iter() { - let mut map: Map = Map::new(); + let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); map_mut.insert(15, "fizzbuzz"); map_mut.insert(5, "buzz"); @@ -561,7 +561,7 @@ mod tests { #[test] fn test_overwrite_insert() { - let mut map: Map = Map::new(); + let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); assert!(map_mut.insert(0, "fizz")); // insert should return false when the key is already present @@ -571,7 +571,7 @@ mod tests { #[test] fn test_extend() { - let mut map: Map = Map::new(); + let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); map_mut.extend([(0, ""); 0]); @@ -588,7 +588,7 @@ mod tests { ] ); - let mut map_2: Map = Map::new(); + let mut map_2: Map = Map::new(); let mut map_2_mut = map_2.as_mut(); map_2_mut.extend([(2, "bing"), (3, "bong")]); @@ -608,7 +608,7 @@ mod tests { #[test] fn test_copy_from() { - let mut map: Map = Map::new(); + let mut map: Map = Map::new(); let mut map_mut = map.as_mut(); map_mut.copy_from([(0, "fizz"), (1, "buzz"), (2, "fizzbuzz")]); @@ -621,7 +621,7 @@ mod tests { ] ); - let mut map_2: Map = Map::new(); + let mut map_2: Map = Map::new(); let mut map_2_mut = map_2.as_mut(); map_2_mut.copy_from([(2, "bing"), (3, "bong")]); @@ -651,12 +651,12 @@ mod tests { macro_rules! gen_proto_keys { ($($key_t:ty),*) => { $( - gen_proto_values!($key_t, f32, f64, i32, u32, i64, bool, ProtoStr, [u8]); + gen_proto_values!($key_t, f32, f64, i32, u32, i64, bool, ProtoString, ProtoBytes); )* } } - gen_proto_keys!(i32, u32, i64, u64, bool, ProtoStr); + gen_proto_keys!(i32, u32, i64, u64, bool, ProtoString); } #[test] diff --git a/rust/proxied.rs b/rust/proxied.rs index d3fa32d687..c6aed93f29 100644 --- a/rust/proxied.rs +++ b/rust/proxied.rs @@ -224,7 +224,6 @@ pub trait IntoProxied { mod tests { use super::*; use googletest::prelude::*; - use std::borrow::Cow; #[derive(Debug, Default, PartialEq)] struct MyProxied { diff --git a/rust/shared.rs b/rust/shared.rs index 7cf07ed970..eb8fd21479 100644 --- a/rust/shared.rs +++ b/rust/shared.rs @@ -30,7 +30,7 @@ pub mod __public { pub use crate::repeated::{ ProxiedInRepeated, Repeated, RepeatedIter, RepeatedMut, RepeatedView, }; - pub use crate::string::ProtoStr; + pub use crate::string::{ProtoBytes, ProtoStr, ProtoString}; pub use crate::{ParseError, SerializeError}; } pub use __public::*; diff --git a/rust/string.rs b/rust/string.rs index 9272c8d0f0..589721c42e 100644 --- a/rust/string.rs +++ b/rust/string.rs @@ -10,23 +10,103 @@ #![allow(unused)] use crate::__internal::Private; -use crate::__runtime::{PtrAndLen, RawMessage}; -use crate::{Mut, MutProxied, MutProxy, Optional, Proxied, View, ViewProxy}; +use crate::__runtime::{InnerProtoString, PtrAndLen, RawMessage}; +use crate::{IntoProxied, Mut, MutProxied, MutProxy, Optional, Proxied, View, ViewProxy}; use std::borrow::Cow; use std::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd}; use std::convert::{AsMut, AsRef}; +use std::ffi::{OsStr, OsString}; use std::fmt; use std::hash::{Hash, Hasher}; use std::iter; use std::ops::{Deref, DerefMut}; +use std::ptr; +use std::rc::Rc; +use std::sync::Arc; use utf8::Utf8Chunks; -impl Proxied for [u8] { +pub struct ProtoBytes { + pub(crate) inner: InnerProtoString, +} + +impl AsRef<[u8]> for ProtoBytes { + fn as_ref(&self) -> &[u8] { + self.inner.as_bytes() + } +} + +impl From<&[u8]> for ProtoBytes { + fn from(v: &[u8]) -> ProtoBytes { + ProtoBytes { inner: InnerProtoString::from(v) } + } +} + +impl From<&[u8; N]> for ProtoBytes { + fn from(v: &[u8; N]) -> ProtoBytes { + ProtoBytes { inner: InnerProtoString::from(v.as_ref()) } + } +} + +impl Proxied for ProtoBytes { type View<'msg> = &'msg [u8]; } +impl IntoProxied for ProtoBytes { + fn into_proxied(self, _private: Private) -> ProtoBytes { + self + } +} + +impl IntoProxied for &[u8] { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(self) + } +} + +impl IntoProxied for &[u8; N] { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(self.as_ref()) + } +} + +impl IntoProxied for Vec { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(AsRef::<[u8]>::as_ref(&self)) + } +} + +impl IntoProxied for &Vec { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(AsRef::<[u8]>::as_ref(self)) + } +} + +impl IntoProxied for Box<[u8]> { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(AsRef::<[u8]>::as_ref(&self)) + } +} + +impl IntoProxied for Cow<'_, [u8]> { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(AsRef::<[u8]>::as_ref(&self)) + } +} + +impl IntoProxied for Rc<[u8]> { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(AsRef::<[u8]>::as_ref(&self)) + } +} + +impl IntoProxied for Arc<[u8]> { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes::from(AsRef::<[u8]>::as_ref(&self)) + } +} + impl<'msg> ViewProxy<'msg> for &'msg [u8] { - type Proxied = [u8]; + type Proxied = ProtoBytes; fn as_view(&self) -> &[u8] { self @@ -50,6 +130,118 @@ impl From for Utf8Error { } } +/// An owned type representing protobuf `string` field's contents. +/// +/// # UTF-8 +/// +/// Protobuf [docs] state that a `string` field contains UTF-8 encoded text. +/// However, not every runtime enforces this, and the Rust runtime is designed +/// to integrate with other runtimes with FFI, like C++. +/// +/// `ProtoString` represents a string type that is expected to contain valid +/// UTF-8. However, `ProtoString` is not validated, so users must +/// call [`ProtoString::to_string`] to perform a (possibly runtime-elided) UTF-8 +/// validation check. This validation should rarely fail in pure Rust programs, +/// but is necessary to prevent UB when interacting with C++, or other languages +/// with looser restrictions. +/// +/// +/// # `Display` and `ToString` +/// `ProtoString` is ordinarily UTF-8 and so implements `Display`. If there are +/// any invalid UTF-8 sequences, they are replaced with [`U+FFFD REPLACEMENT +/// CHARACTER`]. Because anything implementing `Display` also implements +/// `ToString`, `ProtoString::to_string()` is equivalent to +/// `String::from_utf8_lossy(proto_string.as_bytes()).into_owned()`. +/// +/// [`U+FFFD REPLACEMENT CHARACTER`]: std::char::REPLACEMENT_CHARACTER +pub struct ProtoString { + pub(crate) inner: InnerProtoString, +} + +impl ProtoString { + pub fn as_bytes(&self) -> &[u8] { + self.inner.as_bytes() + } +} + +impl From<&str> for ProtoString { + fn from(v: &str) -> Self { + Self::from(v.as_bytes()) + } +} + +impl From<&[u8]> for ProtoString { + fn from(v: &[u8]) -> Self { + Self { inner: InnerProtoString::from(v) } + } +} + +impl IntoProxied for ProtoString { + fn into_proxied(self, _private: Private) -> ProtoString { + self + } +} + +impl IntoProxied for &str { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(self) + } +} + +impl IntoProxied for &ProtoStr { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(self.as_bytes()) + } +} + +impl IntoProxied for String { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(self.as_str()) + } +} + +impl IntoProxied for &String { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(self.as_bytes()) + } +} + +impl IntoProxied for OsString { + fn into_proxied(self, private: Private) -> ProtoString { + self.as_os_str().into_proxied(private) + } +} + +impl IntoProxied for &OsStr { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(self.as_encoded_bytes()) + } +} + +impl IntoProxied for Box { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(AsRef::::as_ref(&self)) + } +} + +impl IntoProxied for Cow<'_, str> { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(AsRef::::as_ref(&self)) + } +} + +impl IntoProxied for Rc { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(AsRef::::as_ref(&self)) + } +} + +impl IntoProxied for Arc { + fn into_proxied(self, _private: Private) -> ProtoString { + ProtoString::from(AsRef::::as_ref(&self)) + } +} + /// A shared immutable view of a protobuf `string` field's contents. /// /// Like a `str`, it can be cheaply accessed as bytes and @@ -261,12 +453,12 @@ impl Ord for ProtoStr { } } -impl Proxied for ProtoStr { +impl Proxied for ProtoString { type View<'msg> = &'msg ProtoStr; } impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { - type Proxied = ProtoStr; + type Proxied = ProtoString; fn as_view(&self) -> &ProtoStr { self @@ -280,30 +472,6 @@ impl<'msg> ViewProxy<'msg> for &'msg ProtoStr { } } -// TODO: remove after IntoProxied has been implemented for -// ProtoStr. -impl AsRef for String { - fn as_ref(&self) -> &ProtoStr { - ProtoStr::from_str(self.as_str()) - } -} - -// TODO: remove after IntoProxied has been implemented for -// ProtoStr. -impl AsRef for &str { - fn as_ref(&self) -> &ProtoStr { - ProtoStr::from_str(self) - } -} - -// TODO: remove after IntoProxied has been implemented for -// ProtoStr. -impl AsRef for &ProtoStr { - fn as_ref(&self) -> &ProtoStr { - self - } -} - /// Implements `PartialCmp` and `PartialEq` for the `lhs` against the `rhs` /// using `AsRef<[u8]>`. // TODO: consider improving to not require a `<()>` if no generics are diff --git a/rust/test/BUILD b/rust/test/BUILD index 0fedc17d2a..b94f3dc2f1 100644 --- a/rust/test/BUILD +++ b/rust/test/BUILD @@ -358,10 +358,7 @@ rust_cc_proto_library( rust_upb_proto_library( name = "nested_upb_rust_proto", testonly = True, - visibility = [ - "//rust/test/shared:__subpackages__", - "//rust/test/upb:__subpackages__", - ], + visibility = ["//rust/test/shared:__subpackages__"], deps = [":nested_proto"], ) diff --git a/rust/test/shared/accessors_test.rs b/rust/test/shared/accessors_test.rs index 956ff06a6b..f108c889c1 100644 --- a/rust/test/shared/accessors_test.rs +++ b/rust/test/shared/accessors_test.rs @@ -8,7 +8,11 @@ //! Tests covering accessors for singular bool, int32, int64, and bytes fields. use googletest::prelude::*; -use protobuf::Optional; +use protobuf::{Optional, ProtoBytes, ProtoStr, ProtoString}; +use std::borrow::Cow; +use std::ffi::OsString; +use std::rc::Rc; +use std::sync::Arc; use unittest_rust_proto::{test_all_types, TestAllTypes}; #[test] @@ -415,6 +419,48 @@ fn test_optional_bytes_accessors() { assert_that!(msg.optional_bytes_opt(), eq(Optional::Set(&b""[..]))); } +#[test] +fn test_into_proxied_for_bytes() { + let mut msg = TestAllTypes::new(); + + // &[u8] + let bytes: &[u8] = b"first"; + msg.set_optional_bytes(bytes); + assert_that!(msg.optional_bytes(), eq(bytes)); + + // &[u8; N] + msg.set_optional_bytes(b"second"); + assert_that!(msg.optional_bytes(), eq(b"second")); + + // Vec + msg.set_optional_bytes(Vec::from(b"third")); + assert_that!(msg.optional_bytes(), eq(b"third")); + + // ProtoBytes + msg.set_optional_bytes(ProtoBytes::from(b"fourth")); + assert_that!(msg.optional_bytes(), eq(b"fourth")); + + // Box<[u8]> + msg.set_optional_bytes(Box::from(b"fifth".to_owned())); + assert_that!(msg.optional_bytes(), eq(b"fifth")); + + // Cow<[u8]> + msg.set_optional_bytes(Cow::from(b"sixth")); + assert_that!(msg.optional_bytes(), eq(b"sixth")); + + // Rc<[u8]> + msg.set_optional_bytes(Rc::from(b"seventh".to_owned())); + assert_that!(msg.optional_bytes(), eq(b"seventh")); + + // Arc<[u8]> + msg.set_optional_bytes(Arc::from(b"eighth".to_owned())); + assert_that!(msg.optional_bytes(), eq(b"eighth")); + + // &Vec + msg.set_optional_bytes(&Vec::from(b"ninth")); + assert_that!(msg.optional_bytes(), eq(b"ninth")); +} + #[test] fn test_nonempty_default_bytes_accessors() { let mut msg = TestAllTypes::new(); @@ -470,6 +516,55 @@ fn test_optional_string_accessors() { assert_that!(msg.optional_string_opt(), eq(Optional::Unset("".into()))); } +#[test] +fn test_into_proxied_for_string() { + let mut msg = TestAllTypes::new(); + + // &str + msg.set_optional_string("first"); + assert_that!(msg.optional_string(), eq("first")); + + // String + msg.set_optional_string("second".to_string()); + assert_that!(msg.optional_string(), eq("second")); + + // ProtoStr + msg.set_optional_string(ProtoStr::from_str("third")); + assert_that!(msg.optional_string(), eq("third")); + + // ProtoString + msg.set_optional_string(ProtoString::from("fourth")); + assert_that!(msg.optional_string(), eq("fourth")); + + // OsString + msg.set_optional_string(OsString::from("fifth")); + assert_that!(msg.optional_string(), eq("fifth")); + + // OsStr + msg.set_optional_string(OsString::from("sixth").as_os_str()); + assert_that!(msg.optional_string(), eq("sixth")); + + // Box + msg.set_optional_string(Box::from("seventh")); + assert_that!(msg.optional_string(), eq("seventh")); + + // Cow + msg.set_optional_string(Cow::from("eighth")); + assert_that!(msg.optional_string(), eq("eighth")); + + // Rc + msg.set_optional_string(Rc::from("ninth")); + assert_that!(msg.optional_string(), eq("ninth")); + + // Arc + msg.set_optional_string(Arc::from("tenth")); + assert_that!(msg.optional_string(), eq("tenth")); + + // &String + msg.set_optional_string(&"eleventh".to_string()); + assert_that!(msg.optional_string(), eq("eleventh")); +} + #[test] fn test_nonempty_default_string_accessors() { let mut msg = TestAllTypes::new(); diff --git a/rust/test/shared/serialization_test.rs b/rust/test/shared/serialization_test.rs index ab9dd21ca1..e5b614034f 100644 --- a/rust/test/shared/serialization_test.rs +++ b/rust/test/shared/serialization_test.rs @@ -61,7 +61,7 @@ macro_rules! generate_parameterized_serialization_test { msg.set_optional_bool(true); let mut msg2 = [< $type >]::new(); msg2.set_optional_bytes(msg.serialize().unwrap()); - assert_that!(msg2.optional_bytes(), eq(msg.serialize().unwrap().as_ref())); + assert_that!(msg2.optional_bytes(), eq(msg.serialize().unwrap())); } #[test] diff --git a/rust/upb.rs b/rust/upb.rs index 1d5e396131..4b1ab8b0c5 100644 --- a/rust/upb.rs +++ b/rust/upb.rs @@ -9,8 +9,8 @@ use crate::__internal::{Enum, Private}; use crate::{ - Map, MapIter, MapMut, MapView, Mut, ProtoStr, Proxied, ProxiedInMapValue, ProxiedInRepeated, - Repeated, RepeatedMut, RepeatedView, View, ViewProxy, + IntoProxied, Map, MapIter, MapMut, MapView, Mut, ProtoBytes, ProtoStr, ProtoString, Proxied, + ProxiedInMapValue, ProxiedInRepeated, Repeated, RepeatedMut, RepeatedView, View, ViewProxy, }; use core::fmt::Debug; use std::alloc::Layout; @@ -60,6 +60,12 @@ impl ScratchSpace { pub type SerializedData = upb::OwnedArenaBox<[u8]>; +impl IntoProxied for SerializedData { + fn into_proxied(self, _private: Private) -> ProtoBytes { + ProtoBytes { inner: InnerProtoString(self) } + } +} + /// The raw contents of every generated message. #[derive(Debug)] pub struct MessageInner { @@ -144,6 +150,27 @@ fn copy_bytes_in_arena<'msg>(arena: &'msg Arena, val: &'msg [u8]) -> &'msg [u8] } } +/// Kernel-specific owned `string` and `bytes` field type. +pub struct InnerProtoString(OwnedArenaBox<[u8]>); + +impl InnerProtoString { + pub(crate) fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl From<&[u8]> for InnerProtoString { + fn from(val: &[u8]) -> InnerProtoString { + let arena = Arena::new(); + let in_arena_copy = arena.copy_slice_in(val); + // SAFETY: + // - `in_arena_copy` is valid slice that will live for `arena`'s lifetime and + // this is the only reference in the program to it. + // - `in_arena_copy` is a pointer into an allocation on `arena` + InnerProtoString(unsafe { OwnedArenaBox::new(Into::into(in_arena_copy), arena) }) + } +} + /// The raw type-erased version of an owned `Repeated`. #[derive(Debug)] pub struct InnerRepeated { @@ -332,7 +359,7 @@ impl_repeated_primitives!( (u64, u64, uint64_val, upb::CType::UInt64), ); -impl_repeated_bytes!((ProtoStr, upb::CType::String), ([u8], upb::CType::Bytes),); +impl_repeated_bytes!((ProtoString, upb::CType::String), (ProtoBytes, upb::CType::Bytes),); /// Copy the contents of `src` into `dest`. /// @@ -564,18 +591,18 @@ impl_upb_type_conversions_for_scalars!( bool, bool_val, upb::CType::Bool, false; ); -impl UpbTypeConversions for [u8] { +impl UpbTypeConversions for ProtoBytes { fn upb_type() -> upb::CType { upb::CType::Bytes } - fn to_message_value(val: View<'_, [u8]>) -> upb_MessageValue { + fn to_message_value(val: View<'_, ProtoBytes>) -> upb_MessageValue { upb_MessageValue { str_val: val.into() } } unsafe fn to_message_value_copy_if_required( raw_arena: RawArena, - val: View<'_, [u8]>, + val: View<'_, ProtoBytes>, ) -> upb_MessageValue { // SAFETY: The arena memory is not freed due to `ManuallyDrop`. let arena = ManuallyDrop::new(unsafe { Arena::from_raw(raw_arena) }); @@ -584,34 +611,34 @@ impl UpbTypeConversions for [u8] { msg_val } - unsafe fn from_message_value<'msg>(msg: upb_MessageValue) -> View<'msg, [u8]> { + unsafe fn from_message_value<'msg>(msg: upb_MessageValue) -> View<'msg, ProtoBytes> { unsafe { msg.str_val.as_ref() } } } -impl UpbTypeConversions for ProtoStr { +impl UpbTypeConversions for ProtoString { fn upb_type() -> upb::CType { upb::CType::String } - fn to_message_value(val: View<'_, ProtoStr>) -> upb_MessageValue { + fn to_message_value(val: View<'_, ProtoString>) -> upb_MessageValue { upb_MessageValue { str_val: val.as_bytes().into() } } unsafe fn to_message_value_copy_if_required( raw_arena: RawArena, - val: View<'_, ProtoStr>, + val: View<'_, ProtoString>, ) -> upb_MessageValue { // SAFETY: `raw_arena` is valid as promised by the caller unsafe { - <[u8] as UpbTypeConversions>::to_message_value_copy_if_required( + ::to_message_value_copy_if_required( raw_arena, val.as_bytes(), ) } } - unsafe fn from_message_value<'msg>(msg: upb_MessageValue) -> View<'msg, ProtoStr> { + unsafe fn from_message_value<'msg>(msg: upb_MessageValue) -> View<'msg, ProtoString> { unsafe { ProtoStr::from_utf8_unchecked(msg.str_val.as_ref()) } } } @@ -734,13 +761,13 @@ macro_rules! impl_ProxiedInMapValue_for_key_types { ($($t:ty),*) => { $( impl_ProxiedInMapValue_for_non_generated_value_types!( - $t ; f32, f64, i32, u32, i64, u64, bool, ProtoStr, [u8] + $t ; f32, f64, i32, u32, i64, u64, bool, ProtoString, ProtoBytes ); )* } } -impl_ProxiedInMapValue_for_key_types!(i32, u32, i64, u64, bool, ProtoStr); +impl_ProxiedInMapValue_for_key_types!(i32, u32, i64, u64, bool, ProtoString); /// `upb_Map_Insert`, but returns a `bool` for whether insert occurred. /// diff --git a/rust/upb/BUILD b/rust/upb/BUILD index 563794994b..83322121d1 100644 --- a/rust/upb/BUILD +++ b/rust/upb/BUILD @@ -45,6 +45,7 @@ cc_library( deps = [ "//upb:mem", "//upb:message", + "//upb:message_compare", "//upb:message_copy", "//upb/mini_table", ], diff --git a/rust/upb/lib.rs b/rust/upb/lib.rs index 643018afe0..9d3490d73a 100644 --- a/rust/upb/lib.rs +++ b/rust/upb/lib.rs @@ -22,7 +22,7 @@ pub use map::{ mod message; pub use message::{ - upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_New, + upb_Message, upb_Message_DeepClone, upb_Message_DeepCopy, upb_Message_IsEqual, upb_Message_New, upb_Message_SetBaseField, RawMessage, }; diff --git a/rust/upb/message.rs b/rust/upb/message.rs index a4159a1fb6..1aad3a4745 100644 --- a/rust/upb/message.rs +++ b/rust/upb/message.rs @@ -28,4 +28,11 @@ extern "C" { mini_table: *const upb_MiniTableField, val: *const std::ffi::c_void, ); + + pub fn upb_Message_IsEqual( + m1: RawMessage, + m2: RawMessage, + mini_table: *const upb_MiniTable, + options: i32, + ) -> bool; } diff --git a/rust/upb/upb_api.c b/rust/upb/upb_api.c index e6d508db1d..8842aa6d12 100644 --- a/rust/upb/upb_api.c +++ b/rust/upb/upb_api.c @@ -10,11 +10,12 @@ #define UPB_BUILD_API +#include "upb/message/accessors.h" // IWYU pragma: keep #include "upb/mem/arena.h" // IWYU pragma: keep #include "upb/message/array.h" // IWYU pragma: keep +#include "upb/message/compare.h" // IWYU pragma: keep #include "upb/message/copy.h" // IWYU pragma: keep #include "upb/message/map.h" // IWYU pragma: keep -#include "upb/message/accessors.h" // IWYU pragma: keep #include "upb/mini_table/message.h" // IWYU pragma: keep const size_t __rust_proto_kUpb_Map_Begin = kUpb_Map_Begin; diff --git a/rust/upb/wire.rs b/rust/upb/wire.rs index aa335cc020..7660c84e9c 100644 --- a/rust/upb/wire.rs +++ b/rust/upb/wire.rs @@ -1,4 +1,4 @@ -use crate::{upb_ExtensionRegistry, upb_MiniTable, Arena, OwnedArenaBox, RawArena, RawMessage}; +use crate::{upb_ExtensionRegistry, upb_MiniTable, Arena, RawArena, RawMessage}; use std::ptr::NonNull; // LINT.IfChange(encode_status) @@ -26,14 +26,23 @@ pub enum DecodeStatus { } // LINT.ThenChange() +#[repr(i32)] +#[allow(dead_code)] +enum DecodeOption { + AliasString = 1, + CheckRequired = 2, + ExperimentalAllowUnlinked = 4, + AlwaysValidateUtf8 = 8, +} + /// If Err, then EncodeStatus != Ok. /// -/// SAFETY: +/// # Safety /// - `msg` must be associated with `mini_table`. pub unsafe fn encode( msg: RawMessage, mini_table: *const upb_MiniTable, -) -> Result, EncodeStatus> { +) -> Result, EncodeStatus> { let arena = Arena::new(); let mut buf: *mut u8 = std::ptr::null_mut(); let mut len = 0usize; @@ -46,8 +55,7 @@ pub unsafe fn encode( if status == EncodeStatus::Ok { assert!(!buf.is_null()); // EncodeStatus Ok should never return NULL data, even for len=0. // SAFETY: upb guarantees that `buf` is valid to read for `len`. - let slice = NonNull::new_unchecked(std::ptr::slice_from_raw_parts_mut(buf, len)); - Ok(OwnedArenaBox::new(slice, arena)) + Ok((*std::ptr::slice_from_raw_parts(buf, len)).to_vec()) } else { Err(status) } @@ -56,7 +64,7 @@ pub unsafe fn encode( /// Decodes into the provided message (merge semantics). If Err, then /// DecodeStatus != Ok. /// -/// SAFETY: +/// # Safety /// - `msg` must be mutable. /// - `msg` must be associated with `mini_table`. pub unsafe fn decode( @@ -67,11 +75,13 @@ pub unsafe fn decode( ) -> Result<(), DecodeStatus> { let len = buf.len(); let buf = buf.as_ptr(); + let options = DecodeOption::CheckRequired as i32; + // SAFETY: // - `mini_table` is the one associated with `msg` // - `buf` is legally readable for at least `buf_size` bytes. // - `extreg` is null. - let status = upb_Decode(buf, len, msg, mini_table, std::ptr::null(), 0, arena.raw()); + let status = upb_Decode(buf, len, msg, mini_table, std::ptr::null(), options, arena.raw()); match status { DecodeStatus::Ok => Ok(()), _ => Err(status), diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 97caa0ea8e..0b2db2c77b 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -231,6 +231,12 @@ proto_library( deps = [":descriptor_proto"], ) +cc_proto_library( + name = "cpp_features_cc_proto", + visibility = ["//editions:__pkg__"], + deps = [":cpp_features_proto"], +) + ################################################################################ # C++ Runtime Library ################################################################################ diff --git a/src/google/protobuf/compiler/cpp/helpers.cc b/src/google/protobuf/compiler/cpp/helpers.cc index 047a52709e..dbfc5f8e9a 100644 --- a/src/google/protobuf/compiler/cpp/helpers.cc +++ b/src/google/protobuf/compiler/cpp/helpers.cc @@ -215,7 +215,7 @@ internal::field_layout::TransformValidation GetLazyStyle( absl::flat_hash_map MessageVars( const Descriptor* desc) { - absl::string_view prefix = IsMapEntryMessage(desc) ? "" : "_impl_."; + absl::string_view prefix = "_impl_."; return { {"any_metadata", absl::StrCat(prefix, "_any_metadata_")}, {"cached_size", absl::StrCat(prefix, "_cached_size_")}, @@ -540,8 +540,7 @@ std::string FieldName(const FieldDescriptor* field) { } std::string FieldMemberName(const FieldDescriptor* field, bool split) { - absl::string_view prefix = - IsMapEntryMessage(field->containing_type()) ? "" : "_impl_."; + absl::string_view prefix = "_impl_."; absl::string_view split_prefix = split ? "_split_->" : ""; if (field->real_containing_oneof() == nullptr) { return absl::StrCat(prefix, split_prefix, FieldName(field), "_"); @@ -1665,6 +1664,8 @@ bool GetBootstrapBasename(const Options& options, absl::string_view basename, "third_party/protobuf/descriptor"}, {"third_party/protobuf/cpp_features", "third_party/protobuf/cpp_features"}, + {"third_party/java/protobuf/java_features", + "third_party/java/protobuf/java_features_bootstrap"}, {"third_party/protobuf/compiler/plugin", "third_party/protobuf/compiler/plugin"}, {"net/proto2/compiler/proto/profile", diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 61ace77f73..37af63340a 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -1286,6 +1286,7 @@ void MessageGenerator::GenerateMapEntryClassDefinition(io::Printer* p) { &_$classname$_default_instance_); } )cc"); + parse_function_generator_->GenerateDataDecls(p); p->Emit(R"cc( const $superclass$::ClassData* GetClassData() const PROTOBUF_FINAL; static const $superclass$::ClassDataFull _class_data_; @@ -2155,6 +2156,7 @@ void MessageGenerator::GenerateClassMethods(io::Printer* p) { $verify$; $class_data$; )cc"); + parse_function_generator_->GenerateDataDefinitions(p); return; } if (IsAnyMessage(descriptor_)) { @@ -3720,19 +3722,6 @@ void MessageGenerator::GenerateClassData(io::Printer* p) { {"is_initialized", is_initialized}, {"pin_weak_descriptor", pin_weak_descriptor}, {"custom_vtable_methods", custom_vtable_methods}, - {"table", - [&] { - // Map entries use the dynamic parser. - if (IsMapEntryMessage(descriptor_)) { - p->Emit(R"cc( - nullptr, // tc_table - )cc"); - } else { - p->Emit(R"cc( - &_table_.header, - )cc"); - } - }}, {"tracker_on_get_metadata", [&] { if (HasTracker(descriptor_, options_)) { @@ -3752,7 +3741,7 @@ void MessageGenerator::GenerateClassData(io::Printer* p) { const ::$proto_ns$::MessageLite::ClassDataFull $classname$::_class_data_ = { $superclass$::ClassData{ - $table$, + &_table_.header, $on_demand_register_arena_dtor$, $is_initialized$, &$classname$::MergeImpl, diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 40cd4a5fd8..f188e07ca8 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -86,6 +86,7 @@ cc_library( srcs = ["java_features.pb.cc"], hdrs = ["java_features.pb.h"], strip_include_prefix = "/src", + visibility = ["//editions:__pkg__"], deps = [ "//src/google/protobuf", "//src/google/protobuf:arena", diff --git a/src/google/protobuf/compiler/java/field_common.cc b/src/google/protobuf/compiler/java/field_common.cc index cf5cdcbf0a..d8d7255e7b 100644 --- a/src/google/protobuf/compiler/java/field_common.cc +++ b/src/google/protobuf/compiler/java/field_common.cc @@ -13,8 +13,6 @@ namespace protobuf { namespace compiler { namespace java { -std::string GetKotlinPropertyName(std::string capitalized_name); - void SetCommonFieldVariables( const FieldDescriptor* descriptor, const FieldGeneratorInfo* info, absl::flat_hash_map* variables) { @@ -68,25 +66,6 @@ static bool IsUpper(char c) { static char ToLower(char c) { return IsUpper(c) ? c - 'A' + 'a' : c; } -// Returns the name by which the generated Java getters and setters should be -// referenced from Kotlin as properties. In the simplest case, the original name -// is something like `foo_bar`, which gets translated into `getFooBar()` etc, -// and that in turn can be referenced from Kotlin as `fooBar`. -// -// The algorithm for translating proto names into Java getters and setters is -// straightforward. The first letter of each underscore-separated word gets -// uppercased and the underscores are deleted. There are no other changes, so in -// particular if the proto name has a string of capitals then those remain -// as-is. -// -// The algorithm that the Kotlin compiler uses to derive the property name is -// slightly more complicated. If the first character after `get` (etc) is a -// capital and the second isn't, then the property name is just that string with -// its first letter lowercased. So `getFoo` becomes `foo` and `getX` becomes -// `x`. But if there is more than one capital, then all but the last get -// lowercased. So `getHTMLPage` becomes `htmlPage`. If there are only capitals -// then they all get lowercased, so `getID` becomes `id`. -// TODO: move this to a Kotlin-specific location std::string GetKotlinPropertyName(std::string capitalized_name) { // Find the first non-capital. If it is the second character, then we just // need to lowercase the first one. Otherwise we need to lowercase everything diff --git a/src/google/protobuf/compiler/java/field_common.h b/src/google/protobuf/compiler/java/field_common.h index 54e58a0ecb..c29d1c25e9 100644 --- a/src/google/protobuf/compiler/java/field_common.h +++ b/src/google/protobuf/compiler/java/field_common.h @@ -1,6 +1,8 @@ #ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_FIELD_COMMON_H__ #define GOOGLE_PROTOBUF_COMPILER_JAVA_FIELD_COMMON_H__ +#include + #include "google/protobuf/descriptor.h" namespace google { @@ -36,6 +38,27 @@ void PrintExtraFieldInfo( const absl::flat_hash_map& variables, io::Printer* printer); +// Returns the name by which the generated Java getters and setters should be +// referenced from Kotlin as properties. In the simplest case, the original name +// is something like `foo_bar`, which gets translated into `getFooBar()` etc, +// and that in turn can be referenced from Kotlin as `fooBar`. +// +// The algorithm for translating proto names into Java getters and setters is +// straightforward. The first letter of each underscore-separated word gets +// uppercased and the underscores are deleted. There are no other changes, so in +// particular if the proto name has a string of capitals then those remain +// as-is. +// +// The algorithm that the Kotlin compiler uses to derive the property name is +// slightly more complicated. If the first character after `get` (etc) is a +// capital and the second isn't, then the property name is just that string with +// its first letter lowercased. So `getFoo` becomes `foo` and `getX` becomes +// `x`. But if there is more than one capital, then all but the last get +// lowercased. So `getHTMLPage` becomes `htmlPage`. If there are only capitals +// then they all get lowercased, so `getID` becomes `id`. +// TODO: move this to a Kotlin-specific location +std::string GetKotlinPropertyName(std::string capitalized_name); + } // namespace java } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/java/full/message.cc b/src/google/protobuf/compiler/java/full/message.cc index 10abbb90e8..433177312a 100644 --- a/src/google/protobuf/compiler/java/full/message.cc +++ b/src/google/protobuf/compiler/java/full/message.cc @@ -587,15 +587,13 @@ void ImmutableMessageGenerator::GenerateMessageSerializationMethods( if (descriptor_->options().message_set_wire_format()) { printer->Print( "com.google.protobuf.GeneratedMessage\n" - " .ExtendableMessage<$classname$>.ExtensionWriter\n" - " extensionWriter = newMessageSetExtensionWriter();\n", - "classname", name_resolver_->GetImmutableClassName(descriptor_)); + " .ExtendableMessage.ExtensionSerializer\n" + " extensionWriter = newMessageSetExtensionSerializer();\n"); } else { printer->Print( "com.google.protobuf.GeneratedMessage\n" - " .ExtendableMessage<$classname$>.ExtensionWriter\n" - " extensionWriter = newExtensionWriter();\n", - "classname", name_resolver_->GetImmutableClassName(descriptor_)); + " .ExtendableMessage.ExtensionSerializer\n" + " extensionWriter = newExtensionSerializer();\n"); } } diff --git a/src/google/protobuf/compiler/java/generator.h b/src/google/protobuf/compiler/java/generator.h index 53cb616072..1c49b86668 100644 --- a/src/google/protobuf/compiler/java/generator.h +++ b/src/google/protobuf/compiler/java/generator.h @@ -18,8 +18,8 @@ #include #include -#include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/java/java_features.pb.h" +#include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/port.h" diff --git a/src/google/protobuf/compiler/java/internal_helpers.h b/src/google/protobuf/compiler/java/internal_helpers.h index 607669b26a..720a34dccb 100644 --- a/src/google/protobuf/compiler/java/internal_helpers.h +++ b/src/google/protobuf/compiler/java/internal_helpers.h @@ -12,8 +12,8 @@ #ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ #define GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ -#include "google/protobuf/compiler/java/generator.h" #include "google/protobuf/compiler/java/java_features.pb.h" +#include "google/protobuf/compiler/java/generator.h" #include "google/protobuf/compiler/java/names.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" diff --git a/src/google/protobuf/compiler/java/kotlin_generator.h b/src/google/protobuf/compiler/java/kotlin_generator.h index afc4085646..a0d36816fb 100644 --- a/src/google/protobuf/compiler/java/kotlin_generator.h +++ b/src/google/protobuf/compiler/java/kotlin_generator.h @@ -12,8 +12,8 @@ #include -#include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/compiler/java/java_features.pb.h" +#include "google/protobuf/compiler/code_generator.h" #include "google/protobuf/descriptor.pb.h" // Must be included last. diff --git a/src/google/protobuf/compiler/java/lite/message.cc b/src/google/protobuf/compiler/java/lite/message.cc index 81e95242aa..1d240672d8 100644 --- a/src/google/protobuf/compiler/java/lite/message.cc +++ b/src/google/protobuf/compiler/java/lite/message.cc @@ -773,16 +773,17 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl( for (auto& kv : oneofs_) { const OneofDescriptor* oneof = kv.second; + auto oneof_name = context_->GetOneofGeneratorInfo(oneof)->name; printer->Print( "public val $oneof_name$Case: $message$.$oneof_capitalized_name$Case\n" " @JvmName(\"get$oneof_capitalized_name$Case\")\n" - " get() = _builder.get$oneof_capitalized_name$Case()\n\n" + " get() = _builder.$oneof_property_name$Case\n\n" "public fun clear$oneof_capitalized_name$() {\n" " _builder.clear$oneof_capitalized_name$()\n" "}\n", - "oneof_name", context_->GetOneofGeneratorInfo(oneof)->name, - "oneof_capitalized_name", - context_->GetOneofGeneratorInfo(oneof)->capitalized_name, "message", + "oneof_name", oneof_name, "oneof_capitalized_name", + context_->GetOneofGeneratorInfo(oneof)->capitalized_name, + "oneof_property_name", GetKotlinPropertyName(oneof_name), "message", EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true))); } diff --git a/src/google/protobuf/compiler/rust/accessors/singular_string.cc b/src/google/protobuf/compiler/rust/accessors/singular_string.cc index 0962229bdc..43774deff3 100644 --- a/src/google/protobuf/compiler/rust/accessors/singular_string.cc +++ b/src/google/protobuf/compiler/rust/accessors/singular_string.cc @@ -49,7 +49,7 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, {"getter", [&] { ctx.Emit(R"rs( - pub fn $field$($view_self$) -> &$view_lifetime$ $proxied_type$ { + pub fn $field$($view_self$) -> $pb$::View<$view_lifetime$, $proxied_type$> { let view = unsafe { $getter_thunk$(self.raw_msg()).as_ref() }; $transform_view$ })rs"); @@ -58,7 +58,7 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, [&] { if (!field.has_presence()) return; ctx.Emit(R"rs( - pub fn $raw_field_name$_opt($view_self$) -> $pb$::Optional<&$view_lifetime$ $proxied_type$> { + pub fn $raw_field_name$_opt($view_self$) -> $pb$::Optional<$pb$::View<$view_lifetime$, $proxied_type$>> { $pb$::Optional::new( self.$field$(), self.has_$raw_field_name$() @@ -69,13 +69,17 @@ void SingularString::InMsgImpl(Context& ctx, const FieldDescriptor& field, {"setter", [&] { if (accessor_case == AccessorCase::VIEW) return; - ctx.Emit(R"rs( - // TODO: Use IntoProxied once string/bytes types support it. - pub fn set_$raw_field_name$(&mut self, val: impl std::convert::AsRef<$proxied_type$>) { + ctx.Emit({{"as_ref_method", + (field.type() == FieldDescriptor::TYPE_STRING + ? "as_bytes()" + : "as_ref()")}}, + R"rs( + pub fn set_$raw_field_name$(&mut self, val: impl $pb$::IntoProxied<$proxied_type$>) { + let into_proxied = val.into_proxied($pbi$::Private); let string_view: $pbr$::PtrAndLen = $pbr$::copy_bytes_in_arena_if_needed_by_runtime( self.as_mutator_message_ref($pbi$::Private), - val.as_ref().into() + into_proxied.$as_ref_method$, ).into(); unsafe { diff --git a/src/google/protobuf/compiler/rust/message.cc b/src/google/protobuf/compiler/rust/message.cc index 5fb4d01032..68946b795a 100644 --- a/src/google/protobuf/compiler/rust/message.cc +++ b/src/google/protobuf/compiler/rust/message.cc @@ -66,7 +66,7 @@ void MessageSerialize(Context& ctx, const Descriptor& msg) { $serialize_thunk$(self.raw_msg(), &mut serialized_data) }; if success { - Ok(serialized_data) + Ok(serialized_data.into_vec()) } else { Err($pb$::SerializeError) } @@ -939,7 +939,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.msg } - pub fn serialize(&self) -> Result<$pbr$::SerializedData, $pb$::SerializeError> { + pub fn serialize(&self) -> Result, $pb$::SerializeError> { $Msg::serialize$ } @@ -1015,7 +1015,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { self.inner } - pub fn serialize(&self) -> Result<$pbr$::SerializedData, $pb$::SerializeError> { + pub fn serialize(&self) -> Result, $pb$::SerializeError> { $pb$::ViewProxy::as_view(self).serialize() } @@ -1069,7 +1069,7 @@ void GenerateRs(Context& ctx, const Descriptor& msg) { $raw_arena_getter_for_message$ - pub fn serialize(&self) -> Result<$pbr$::SerializedData, $pb$::SerializeError> { + pub fn serialize(&self) -> Result, $pb$::SerializeError> { self.as_view().serialize() } #[deprecated = "Prefer Msg::parse(), or use the new name 'clear_and_parse' to parse into a pre-existing message."] diff --git a/src/google/protobuf/compiler/rust/naming.cc b/src/google/protobuf/compiler/rust/naming.cc index a0dfb961b6..df910264be 100644 --- a/src/google/protobuf/compiler/rust/naming.cc +++ b/src/google/protobuf/compiler/rust/naming.cc @@ -191,9 +191,9 @@ std::string RsTypePath(Context& ctx, const FieldDescriptor& field) { case RustFieldType::DOUBLE: return "f64"; case RustFieldType::BYTES: - return "[u8]"; + return "::__pb::ProtoBytes"; case RustFieldType::STRING: - return "::__pb::ProtoStr"; + return "::__pb::ProtoString"; case RustFieldType::MESSAGE: return GetFullyQualifiedPath(ctx, *field.message_type()); case RustFieldType::ENUM: @@ -449,8 +449,8 @@ PROTOBUF_CONSTINIT const MapKeyType kMapKeyTypes[] = { /*cc_key_t=*/"bool", /*cc_ffi_key_t=*/"bool", /*cc_from_ffi_key_expr=*/"key", /*cc_to_ffi_key_expr=*/"cpp_key"}, - {/*thunk_ident=*/"ProtoStr", - /*rs_key_t=*/"$pb$::ProtoStr", + {/*thunk_ident=*/"ProtoString", + /*rs_key_t=*/"$pb$::ProtoString", /*rs_ffi_key_t=*/"$pbr$::PtrAndLen", /*rs_to_ffi_key_expr=*/"key.as_bytes().into()", /*rs_from_ffi_key_expr=*/ diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index 1c6115de8b..dfabac4168 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -635,13 +635,14 @@ message MessageOptions { } message FieldOptions { + // NOTE: ctype is deprecated. Use `features.(pb.cpp).string_type` instead. // The ctype option instructs the C++ code generator to use a different // representation of the field than it normally would. See the specific // options below. This option is only implemented to support use of // [ctype=CORD] and [ctype=STRING] (the default) on non-repeated fields of - // type "bytes" in the open source release -- sorry, we'll try to include - // other types in a future version! - optional CType ctype = 1 [default = STRING]; + // type "bytes" in the open source release. + // TODO: make ctype actually deprecated. + optional CType ctype = 1 [/*deprecated = true,*/ default = STRING]; enum CType { // Default mode. STRING = 0; diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 763582c05e..0953b12d00 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -560,7 +560,7 @@ class PROTOBUF_EXPORT UntypedMapBase { protected: // 16 bytes is the minimum useful size for the array cache in the arena. - enum { kMinTableSize = 16 / sizeof(void*) }; + enum : map_index_t { kMinTableSize = 16 / sizeof(void*) }; public: Arena* arena() const { return this->alloc_.arena(); } @@ -645,9 +645,7 @@ class PROTOBUF_EXPORT UntypedMapBase { // Return a power of two no less than max(kMinTableSize, n). // Assumes either n < kMinTableSize or n is a power of two. map_index_t TableSize(map_index_t n) { - return n < static_cast(kMinTableSize) - ? static_cast(kMinTableSize) - : n; + return n < kMinTableSize ? kMinTableSize : n; } template @@ -697,7 +695,7 @@ class PROTOBUF_EXPORT UntypedMapBase { } TableEntryPtr* CreateEmptyTable(map_index_t n) { - ABSL_DCHECK_GE(n, map_index_t{kMinTableSize}); + ABSL_DCHECK_GE(n, kMinTableSize); ABSL_DCHECK_EQ(n & (n - 1), 0u); TableEntryPtr* result = AllocFor(alloc_).allocate(n); memset(result, 0, n * sizeof(result[0])); diff --git a/src/google/protobuf/map_entry.h b/src/google/protobuf/map_entry.h index 16145c9b04..71dda48b9f 100644 --- a/src/google/protobuf/map_entry.h +++ b/src/google/protobuf/map_entry.h @@ -95,9 +95,9 @@ class MapEntry : public Message { ~MapEntry() PROTOBUF_OVERRIDE { if (GetArena() != nullptr) return; - Message::_internal_metadata_.template Delete(); - KeyTypeHandler::DeleteNoArena(key_); - ValueTypeHandler::DeleteNoArena(value_); + this->_internal_metadata_.template Delete(); + KeyTypeHandler::DeleteNoArena(_impl_.key_); + ValueTypeHandler::DeleteNoArena(_impl_.value_); } using InternalArenaConstructable_ = void; @@ -107,14 +107,29 @@ class MapEntry : public Message { return Arena::Create(arena); } + struct _Internal; + protected: friend class google::protobuf::Arena; - HasBits<1> _has_bits_{}; - mutable CachedSize _cached_size_{}; + // Field naming follows the convention of generated messages to make code + // sharing easier. + struct { + HasBits<1> _has_bits_{}; + mutable CachedSize _cached_size_{}; + + KeyOnMemory key_{KeyTypeHandler::Constinit()}; + ValueOnMemory value_{ValueTypeHandler::Constinit()}; + } _impl_; +}; - KeyOnMemory key_{KeyTypeHandler::Constinit()}; - ValueOnMemory value_{ValueTypeHandler::Constinit()}; +template +struct MapEntry::_Internal { + static constexpr ::int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MapEntry, _impl_._has_bits_); }; } // namespace internal diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index af5522d6a2..86f1aeb854 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -127,7 +127,7 @@ static constexpr const ::_pb::ServiceDescriptor** const ::uint32_t TableStruct_google_2fprotobuf_2fstruct_2eproto::offsets[] ABSL_ATTRIBUTE_SECTION_VARIABLE( protodesc_cold) = { - PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, _has_bits_), + PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, _impl_._has_bits_), PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -135,8 +135,8 @@ const ::uint32_t ~0u, // no _inlined_string_donated_ ~0u, // no _split_ ~0u, // no sizeof(Split) - PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, key_), - PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, value_), + PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, _impl_.key_), + PROTOBUF_FIELD_OFFSET(::google::protobuf::Struct_FieldsEntry_DoNotUse, _impl_.value_), 0, 1, ~0u, // no _has_bits_ @@ -249,7 +249,7 @@ bool NullValue_IsValid(int value) { const ::google::protobuf::MessageLite::ClassDataFull Struct_FieldsEntry_DoNotUse::_class_data_ = { ::google::protobuf::Message::ClassData{ - nullptr, // tc_table + &_table_.header, nullptr, // OnDemandRegisterArenaDtor nullptr, // IsInitialized &Struct_FieldsEntry_DoNotUse::MergeImpl, @@ -259,7 +259,7 @@ bool NullValue_IsValid(int value) { ::google::protobuf::Message::ClearImpl, ::google::protobuf::Message::ByteSizeLongImpl, ::google::protobuf::Message::_InternalSerializeImpl, #endif // PROTOBUF_CUSTOM_VTABLE - PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _cached_size_), + PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _impl_._cached_size_), false, }, &Struct_FieldsEntry_DoNotUse::kDescriptorMethods, @@ -271,6 +271,49 @@ bool NullValue_IsValid(int value) { ::google::protobuf::internal::PrefetchToLocalCache(_class_data_.tc_table); return _class_data_.base(); } +PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 +const ::_pbi::TcParseTable<1, 2, 1, 46, 2> Struct_FieldsEntry_DoNotUse::_table_ = { + { + PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _impl_._has_bits_), + 0, // no _extensions_ + 2, 8, // max_field_number, fast_idx_mask + offsetof(decltype(_table_), field_lookup_table), + 4294967292, // skipmap + offsetof(decltype(_table_), field_entries), + 2, // num_field_entries + 1, // num_aux_entries + offsetof(decltype(_table_), aux_entries), + &_Struct_FieldsEntry_DoNotUse_default_instance_._instance, + nullptr, // post_loop_handler + ::_pbi::TcParser::DiscardEverythingFallback, // fallback + #ifdef PROTOBUF_PREFETCH_PARSE_TABLE + ::_pbi::TcParser::GetTable<::google::protobuf::Struct_FieldsEntry_DoNotUse>(), // to_prefetch + #endif // PROTOBUF_PREFETCH_PARSE_TABLE + }, {{ + // .google.protobuf.Value value = 2; + {::_pbi::TcParser::FastMtS1, + {18, 0, 0, PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _impl_.value_)}}, + // string key = 1; + {::_pbi::TcParser::FastUS1, + {10, 63, 0, PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _impl_.key_)}}, + }}, {{ + 65535, 65535 + }}, {{ + // string key = 1; + {PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _impl_.key_), -1, 0, + (0 | ::_fl::kFcSingular | ::_fl::kUtf8String | ::_fl::kRepAString)}, + // .google.protobuf.Value value = 2; + {PROTOBUF_FIELD_OFFSET(Struct_FieldsEntry_DoNotUse, _impl_.value_), _Internal::kHasBitsOffset + 0, 0, + (0 | ::_fl::kFcOptional | ::_fl::kMessage | ::_fl::kTvTable)}, + }}, {{ + {::_pbi::TcParser::GetTable<::google::protobuf::Value>()}, + }}, {{ + "\42\3\0\0\0\0\0\0" + "google.protobuf.Struct.FieldsEntry" + "key" + }}, +}; + // =================================================================== class Struct::_Internal { diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 2fc87821fa..f8fc679eb3 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -488,6 +488,11 @@ class Struct_FieldsEntry_DoNotUse final return reinterpret_cast( &_Struct_FieldsEntry_DoNotUse_default_instance_); } +friend class ::google::protobuf::internal::TcParser; +static const ::google::protobuf::internal::TcParseTable< + 1, 2, 1, + 46, 2> + _table_; const ::google::protobuf::Message::ClassData* GetClassData() const PROTOBUF_FINAL; static const ::google::protobuf::Message::ClassDataFull _class_data_; friend struct ::TableStruct_google_2fprotobuf_2fstruct_2eproto; diff --git a/upb/cmake/google/protobuf/descriptor.upb_minitable.c b/upb/cmake/google/protobuf/descriptor.upb_minitable.c index 05bece8f65..08fa451f75 100644 --- a/upb/cmake/google/protobuf/descriptor.upb_minitable.c +++ b/upb/cmake/google/protobuf/descriptor.upb_minitable.c @@ -13,7 +13,7 @@ // Must be last. #include "upb/port/def.inc" -extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty); +extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken); static const upb_MiniTableSubInternal google_protobuf_FileDescriptorSet_submsgs[1] = { {.UPB_PRIVATE(submsg) = &google__protobuf__FileDescriptorProto_msg_init_ptr}, }; diff --git a/upb/mini_table/internal/message.c b/upb/mini_table/internal/message.c index f278b36577..2f23df4118 100644 --- a/upb/mini_table/internal/message.c +++ b/upb/mini_table/internal/message.c @@ -14,7 +14,12 @@ // Must be last. #include "upb/port/def.inc" -// A MiniTable for an empty message, used for unlinked sub-messages. +// A MiniTable for an empty message, used for unlinked sub-messages that are +// built via MiniDescriptors. Messages that use this MiniTable may possibly +// be linked later, in which case this MiniTable will be replaced with a real +// one. This pattern is known as "dynamic tree shaking", and it introduces +// complication because sub-messages may either be the "empty" type or the +// "real" type. A tagged bit indicates the difference. const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { .UPB_PRIVATE(subs) = NULL, .UPB_PRIVATE(fields) = NULL, @@ -25,3 +30,18 @@ const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty) = { .UPB_PRIVATE(table_mask) = -1, .UPB_PRIVATE(required_count) = 0, }; + +// A MiniTable for a statically tree shaken message. Messages that use this +// MiniTable are guaranteed to remain unlinked; unlike the empty message, this +// MiniTable is never replaced, which greatly simplifies everything, because the +// type of a sub-message is always known, without consulting a tagged bit. +const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken) = { + .UPB_PRIVATE(subs) = NULL, + .UPB_PRIVATE(fields) = NULL, + .UPB_PRIVATE(size) = sizeof(struct upb_Message), + .UPB_PRIVATE(field_count) = 0, + .UPB_PRIVATE(ext) = kUpb_ExtMode_NonExtendable, + .UPB_PRIVATE(dense_below) = 0, + .UPB_PRIVATE(table_mask) = -1, + .UPB_PRIVATE(required_count) = 0, +}; diff --git a/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c b/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c index 05bece8f65..08fa451f75 100644 --- a/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c +++ b/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.c @@ -13,7 +13,7 @@ // Must be last. #include "upb/port/def.inc" -extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty); +extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken); static const upb_MiniTableSubInternal google_protobuf_FileDescriptorSet_submsgs[1] = { {.UPB_PRIVATE(submsg) = &google__protobuf__FileDescriptorProto_msg_init_ptr}, }; diff --git a/upb/test/BUILD b/upb/test/BUILD index 98d00e1236..d85f03e9ca 100644 --- a/upb/test/BUILD +++ b/upb/test/BUILD @@ -68,7 +68,7 @@ proto_library( name = "test_proto", testonly = 1, srcs = ["test.proto"], - visibility = ["//upb:__subpackages__"], + visibility = ["//visibility:private"], ) upb_minitable_proto_library( diff --git a/upb_generator/bootstrap_compiler.bzl b/upb_generator/bootstrap_compiler.bzl index 4a3bd8ccdd..e92fbafe75 100644 --- a/upb_generator/bootstrap_compiler.bzl +++ b/upb_generator/bootstrap_compiler.bzl @@ -126,7 +126,14 @@ def _cmake_staleness_test(name, base_dir, src_files, proto_lib_deps, **kwargs): name = name + "_copy_gencode_%d" % genrule, outs = ["generated_sources/" + src], srcs = [name, name + "_minitable"], - cmd = "mkdir -p $(@D); for src in $(SRCS); do cp -f $$src $(@D) || echo 'copy failed!'; done", + cmd = """ + mkdir -p $(@D) + for src in $(SRCS); do + if [[ $$src == *%s ]]; then + cp -f $$src $(@D) || echo 'copy failed!' + fi + done + """ % src[src.rfind("/"):], ) # Keep bazel gencode in sync with our checked-in sources needed for cmake builds. diff --git a/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c b/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c index fab151bf1f..0aaf0cd87b 100644 --- a/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c +++ b/upb_generator/cmake/google/protobuf/compiler/plugin.upb_minitable.c @@ -14,7 +14,7 @@ // Must be last. #include "upb/port/def.inc" -extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_Empty); +extern const struct upb_MiniTable UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken); static const upb_MiniTableField google_protobuf_compiler_Version__fields[4] = { {1, 12, 64, kUpb_NoSub, 5, (int)kUpb_FieldMode_Scalar | ((int)kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)}, {2, 16, 65, kUpb_NoSub, 5, (int)kUpb_FieldMode_Scalar | ((int)kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)}, diff --git a/upb_generator/protoc-gen-upb_minitable.cc b/upb_generator/protoc-gen-upb_minitable.cc index 2ed83dd1c1..dbc959f82f 100644 --- a/upb_generator/protoc-gen-upb_minitable.cc +++ b/upb_generator/protoc-gen-upb_minitable.cc @@ -367,8 +367,8 @@ void WriteMessage(upb::MessageDefPtr message, const DefPoolPair& pools, IsCrossFile(field)) { if (seen.insert(pools.GetMiniTable64(field.message_type())).second) { output( - "__attribute__((weak)) const upb_MiniTable* $0 = " - "&UPB_PRIVATE(_kUpb_MiniTable_Empty);\n", + "__attribute__((weak)) const upb_MiniTable* $0 =" + " &UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken);\n", MessagePtrName(field.message_type())); } } @@ -571,7 +571,7 @@ void WriteMiniTableSourceIncludes(upb::FileDefPtr file, Output& output) { output( "extern const struct upb_MiniTable " - "UPB_PRIVATE(_kUpb_MiniTable_Empty);\n"); + "UPB_PRIVATE(_kUpb_MiniTable_StaticallyTreeShaken);\n"); } void WriteMiniTableSource(const DefPoolPair& pools, upb::FileDefPtr file,